|
|
|
|
# Prerequisites for Windows:
|
|
|
|
|
# This cmake build is for Windows 64-bit only.
|
|
|
|
|
#
|
|
|
|
|
# Prerequisites:
|
|
|
|
|
# You must have at least Visual Studio 2015 Update 3. Start the Developer Command Prompt window that is a part of Visual Studio installation.
|
|
|
|
|
# Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set.
|
|
|
|
|
# You must have git.exe in your %PATH% environment variable.
|
|
|
|
|
#
|
|
|
|
|
# To build Rocksdb for Windows is as easy as 1-2-3-4-5:
|
|
|
|
|
#
|
|
|
|
|
# 1. Update paths to third-party libraries in thirdparty.inc file
|
|
|
|
|
# 2. Create a new directory for build artifacts
|
|
|
|
|
# mkdir build
|
|
|
|
|
# cd build
|
|
|
|
|
# 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
|
|
|
|
|
# See thirdparty.inc for more information.
|
|
|
|
|
# sample command: cmake -G "Visual Studio 15 Win64" -DWITH_GFLAGS=1 -DWITH_SNAPPY=1 -DWITH_JEMALLOC=1 -DWITH_JNI=1 ..
|
|
|
|
|
# 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
|
|
|
|
|
# or simply /m to use all avail cores)
|
|
|
|
|
# msbuild rocksdb.sln
|
|
|
|
|
#
|
|
|
|
|
# rocksdb.sln build features exclusions of test only code in Release. If you build ALL_BUILD then everything
|
|
|
|
|
# will be attempted but test only code does not build in Release mode.
|
|
|
|
|
#
|
|
|
|
|
# 5. And release mode (/m[:<N>] is also supported)
|
|
|
|
|
# msbuild rocksdb.sln /p:Configuration=Release
|
|
|
|
|
#
|
|
|
|
|
# Linux:
|
|
|
|
|
#
|
|
|
|
|
# 1. Install a recent toolchain such as devtoolset-3 if you're on a older distro. C++11 required.
|
|
|
|
|
# 2. mkdir build; cd build
|
|
|
|
|
# 3. cmake ..
|
|
|
|
|
# 4. make -j
|
|
|
|
|
|
|
|
|
|
cmake_minimum_required(VERSION 3.5.1)
|
|
|
|
|
|
|
|
|
|
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")
|
|
|
|
|
include(ReadVersion)
|
|
|
|
|
get_rocksdb_version(rocksdb_VERSION)
|
|
|
|
|
project(rocksdb
|
|
|
|
|
VERSION ${rocksdb_VERSION}
|
|
|
|
|
LANGUAGES CXX C ASM)
|
|
|
|
|
|
|
|
|
|
if(POLICY CMP0042)
|
|
|
|
|
cmake_policy(SET CMP0042 NEW)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
find_program(CCACHE_FOUND ccache)
|
|
|
|
|
if(CCACHE_FOUND)
|
|
|
|
|
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
|
|
|
|
|
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
|
|
|
|
|
endif(CCACHE_FOUND)
|
|
|
|
|
|
|
|
|
|
option(WITH_JEMALLOC "build with JeMalloc" OFF)
|
|
|
|
|
option(WITH_SNAPPY "build with SNAPPY" OFF)
|
|
|
|
|
option(WITH_LZ4 "build with lz4" OFF)
|
|
|
|
|
option(WITH_ZLIB "build with zlib" OFF)
|
|
|
|
|
option(WITH_ZSTD "build with zstd" OFF)
|
|
|
|
|
option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, regardles of the system code page" OFF)
|
|
|
|
|
if (WITH_WINDOWS_UTF8_FILENAMES)
|
|
|
|
|
add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
|
|
|
|
|
endif()
|
|
|
|
|
# third-party/folly is only validated to work on Linux and Windows for now.
|
|
|
|
|
# So only turn it on there by default.
|
|
|
|
|
if(CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_SYSTEM_NAME MATCHES "Windows")
|
|
|
|
|
option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" ON)
|
|
|
|
|
else()
|
|
|
|
|
option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
|
|
|
|
|
endif()
|
|
|
|
|
if(MSVC)
|
|
|
|
|
# Defaults currently different for GFLAGS.
|
|
|
|
|
# We will address find_package work a little later
|
|
|
|
|
option(WITH_GFLAGS "build with GFlags" OFF)
|
|
|
|
|
option(WITH_XPRESS "build with windows built in compression" OFF)
|
|
|
|
|
include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
|
|
|
|
|
else()
|
|
|
|
|
if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
|
|
|
|
|
# FreeBSD has jemalloc as default malloc
|
|
|
|
|
# but it does not have all the jemalloc files in include/...
|
|
|
|
|
set(WITH_JEMALLOC ON)
|
|
|
|
|
else()
|
|
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
find_package(JeMalloc REQUIRED)
|
|
|
|
|
add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS JeMalloc::JeMalloc)
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
# No config file for this
|
|
|
|
|
option(WITH_GFLAGS "build with GFlags" ON)
|
|
|
|
|
if(WITH_GFLAGS)
|
|
|
|
|
find_package(gflags)
|
|
|
|
|
if(gflags_FOUND)
|
|
|
|
|
add_definitions(-DGFLAGS=1)
|
|
|
|
|
include_directories(${gflags_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${gflags_LIBRARIES})
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(WITH_SNAPPY)
|
|
|
|
|
find_package(snappy REQUIRED)
|
|
|
|
|
add_definitions(-DSNAPPY)
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS snappy::snappy)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(WITH_ZLIB)
|
|
|
|
|
find_package(ZLIB REQUIRED)
|
|
|
|
|
add_definitions(-DZLIB)
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ZLIB::ZLIB)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_BZ2 "build with bzip2" OFF)
|
|
|
|
|
if(WITH_BZ2)
|
|
|
|
|
find_package(BZip2 REQUIRED)
|
|
|
|
|
add_definitions(-DBZIP2)
|
|
|
|
|
if(BZIP2_INCLUDE_DIRS)
|
|
|
|
|
include_directories(${BZIP2_INCLUDE_DIRS})
|
|
|
|
|
else()
|
|
|
|
|
include_directories(${BZIP2_INCLUDE_DIR})
|
|
|
|
|
endif()
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES})
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(WITH_LZ4)
|
|
|
|
|
find_package(lz4 REQUIRED)
|
|
|
|
|
add_definitions(-DLZ4)
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS lz4::lz4)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(WITH_ZSTD)
|
|
|
|
|
find_package(zstd REQUIRED)
|
|
|
|
|
add_definitions(-DZSTD)
|
|
|
|
|
include_directories(${ZSTD_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS zstd::zstd)
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
string(TIMESTAMP TS "%Y/%m/%d %H:%M:%S" UTC)
|
|
|
|
|
set(GIT_DATE_TIME "${TS}" CACHE STRING "the time we first built rocksdb")
|
|
|
|
|
|
|
|
|
|
find_package(Git)
|
|
|
|
|
|
|
|
|
|
if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
|
|
|
|
|
if(WIN32)
|
|
|
|
|
execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
|
|
|
|
|
else()
|
|
|
|
|
execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
|
|
|
|
|
endif()
|
|
|
|
|
else()
|
|
|
|
|
set(GIT_SHA 0)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
|
|
|
|
|
|
|
|
|
|
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
option(WITH_MD_LIBRARY "build with MD" ON)
|
|
|
|
|
if(WIN32 AND MSVC)
|
|
|
|
|
if(WITH_MD_LIBRARY)
|
|
|
|
|
set(RUNTIME_LIBRARY "MD")
|
|
|
|
|
else()
|
|
|
|
|
set(RUNTIME_LIBRARY "MT")
|
|
|
|
|
endif()
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
|
|
|
|
|
configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
|
|
|
|
|
add_library(build_version OBJECT ${BUILD_VERSION_CC})
|
|
|
|
|
target_include_directories(build_version PRIVATE
|
|
|
|
|
${CMAKE_CURRENT_SOURCE_DIR}/util)
|
|
|
|
|
if(MSVC)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
|
|
|
|
|
else()
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
|
|
|
|
|
if(MINGW)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
|
|
|
|
|
add_definitions(-D_POSIX_C_SOURCE=1)
|
|
|
|
|
endif()
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
|
|
|
|
|
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
|
|
|
|
|
include(CheckCXXCompilerFlag)
|
|
|
|
|
CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
|
|
|
|
|
if(HAVE_OMIT_LEAF_FRAME_POINTER)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
include(CheckCCompilerFlag)
|
|
|
|
|
if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
|
|
|
|
|
CHECK_C_COMPILER_FLAG("-maltivec" HAS_ALTIVEC)
|
|
|
|
|
if(HAS_ALTIVEC)
|
|
|
|
|
message(STATUS " HAS_ALTIVEC yes")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maltivec")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maltivec")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power8")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power8")
|
|
|
|
|
endif(HAS_ALTIVEC)
|
|
|
|
|
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
|
|
|
|
|
|
|
|
|
|
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
|
|
|
|
|
CHECK_C_COMPILER_FLAG("-march=armv8-a+crc+crypto" HAS_ARMV8_CRC)
|
|
|
|
|
if(HAS_ARMV8_CRC)
|
|
|
|
|
message(STATUS " HAS_ARMV8_CRC yes")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
|
|
|
|
|
endif(HAS_ARMV8_CRC)
|
|
|
|
|
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
|
|
|
|
|
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
option(PORTABLE "build a portable binary" OFF)
|
|
|
|
|
option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
|
|
|
|
|
if(PORTABLE)
|
|
|
|
|
# MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
|
|
|
|
|
# is available, it is available by default.
|
|
|
|
|
if(FORCE_SSE42 AND NOT MSVC)
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
7 years ago
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -mpclmul")
|
|
|
|
|
endif()
|
|
|
|
|
else()
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
if(MSVC)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
|
|
|
|
|
else()
|
|
|
|
|
if(NOT HAVE_POWER8 AND NOT HAS_ARMV8_CRC)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
include(CheckCXXSourceCompiles)
|
|
|
|
|
if(NOT MSVC)
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
7 years ago
|
|
|
set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
|
|
|
|
|
endif()
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
|
|
|
|
#include <cstdint>
|
|
|
|
|
#include <nmmintrin.h>
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
7 years ago
|
|
|
#include <wmmintrin.h>
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
int main() {
|
|
|
|
|
volatile uint32_t x = _mm_crc32_u32(0, 0);
|
Port 3 way SSE4.2 crc32c implementation from Folly
Summary:
**# Summary**
RocksDB uses SSE crc32 intrinsics to calculate the crc32 values but it does it in single way fashion (not pipelined on single CPU core). Intel's whitepaper () published an algorithm that uses 3-way pipelining for the crc32 intrinsics, then use pclmulqdq intrinsic to combine the values. Because pclmulqdq has overhead on its own, this algorithm will show perf gains on buffers larger than 216 bytes, which makes RocksDB a perfect user, since most of the buffers RocksDB call crc32c on is over 4KB. Initial db_bench show tremendous CPU gain.
This change uses the 3-way SSE algorithm by default. The old SSE algorithm is now behind a compiler tag NO_THREEWAY_CRC32C. If user compiles the code with NO_THREEWAY_CRC32C=1 then the old SSE Crc32c algorithm would be used. If the server does not have SSE4.2 at the run time the slow way (Non SSE) will be used.
**# Performance Test Results**
We ran the FillRandom and ReadRandom benchmarks in db_bench. ReadRandom is the point of interest here since it calculates the CRC32 for the in-mem buffers. We did 3 runs for each algorithm.
Before this change the CRC32 value computation takes about 11.5% of total CPU cost, and with the new 3-way algorithm it reduced to around 4.5%. The overall throughput also improved from 25.53MB/s to 27.63MB/s.
1) ReadRandom in db_bench overall metrics
PER RUN
Algorithm | run | micros/op | ops/sec |Throughput (MB/s)
3-way | 1 | 4.143 | 241387 | 26.7
3-way | 2 | 3.775 | 264872 | 29.3
3-way | 3 | 4.116 | 242929 | 26.9
FastCrc32c|1 | 4.037 | 247727 | 27.4
FastCrc32c|2 | 4.648 | 215166 | 23.8
FastCrc32c|3 | 4.352 | 229799 | 25.4
AVG
Algorithm | Average of micros/op | Average of ops/sec | Average of Throughput (MB/s)
3-way | 4.01 | 249,729 | 27.63
FastCrc32c | 4.35 | 230,897 | 25.53
2) Crc32c computation CPU cost (inclusive samples percentage)
PER RUN
Implementation | run | TotalSamples | Crc32c percentage
3-way | 1 | 4,572,250,000 | 4.37%
3-way | 2 | 3,779,250,000 | 4.62%
3-way | 3 | 4,129,500,000 | 4.48%
FastCrc32c | 1 | 4,663,500,000 | 11.24%
FastCrc32c | 2 | 4,047,500,000 | 12.34%
FastCrc32c | 3 | 4,366,750,000 | 11.68%
**# Test Plan**
make -j64 corruption_test && ./corruption_test
By default it uses 3-way SSE algorithm
NO_THREEWAY_CRC32C=1 make -j64 corruption_test && ./corruption_test
make clean && DEBUG_LEVEL=0 make -j64 db_bench
make clean && DEBUG_LEVEL=0 NO_THREEWAY_CRC32C=1 make -j64 db_bench
Closes https://github.com/facebook/rocksdb/pull/3173
Differential Revision: D6330882
Pulled By: yingsu00
fbshipit-source-id: 8ec3d89719533b63b536a736663ca6f0dd4482e9
7 years ago
|
|
|
const auto a = _mm_set_epi64x(0, 0);
|
|
|
|
|
const auto b = _mm_set_epi64x(0, 0);
|
|
|
|
|
const auto c = _mm_clmulepi64_si128(a, b, 0x00);
|
|
|
|
|
auto d = _mm_cvtsi128_si64(c);
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
}
|
|
|
|
|
" HAVE_SSE42)
|
|
|
|
|
unset(CMAKE_REQUIRED_FLAGS)
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
if(HAVE_SSE42)
|
|
|
|
|
add_definitions(-DHAVE_SSE42)
|
|
|
|
|
add_definitions(-DHAVE_PCLMUL)
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
elseif(FORCE_SSE42)
|
|
|
|
|
message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
|
|
|
|
#if defined(_MSC_VER) && !defined(__thread)
|
|
|
|
|
#define __thread __declspec(thread)
|
|
|
|
|
#endif
|
|
|
|
|
int main() {
|
|
|
|
|
static __thread int tls;
|
|
|
|
|
}
|
|
|
|
|
" HAVE_THREAD_LOCAL)
|
|
|
|
|
if(HAVE_THREAD_LOCAL)
|
|
|
|
|
add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
|
|
|
|
|
if(FAIL_ON_WARNINGS)
|
|
|
|
|
if(MSVC)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX")
|
|
|
|
|
else() # assume GCC
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_ASAN "build with ASAN" OFF)
|
|
|
|
|
if(WITH_ASAN)
|
|
|
|
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
|
|
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
message(FATAL "ASAN does not work well with JeMalloc")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_TSAN "build with TSAN" OFF)
|
|
|
|
|
if(WITH_TSAN)
|
|
|
|
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -pie")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC")
|
|
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
message(FATAL "TSAN does not work well with JeMalloc")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_UBSAN "build with UBSAN" OFF)
|
|
|
|
|
if(WITH_UBSAN)
|
|
|
|
|
add_definitions(-DROCKSDB_UBSAN_RUN)
|
|
|
|
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
|
|
|
|
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined")
|
|
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
message(FATAL "UBSAN does not work well with JeMalloc")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_NUMA "build with NUMA policy support" OFF)
|
|
|
|
|
if(WITH_NUMA)
|
|
|
|
|
find_package(NUMA REQUIRED)
|
|
|
|
|
add_definitions(-DNUMA)
|
|
|
|
|
include_directories(${NUMA_INCLUDE_DIR})
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS NUMA::NUMA)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_TBB "build with Threading Building Blocks (TBB)" OFF)
|
|
|
|
|
if(WITH_TBB)
|
|
|
|
|
find_package(TBB REQUIRED)
|
|
|
|
|
add_definitions(-DTBB)
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS TBB::TBB)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
# Stall notifications eat some performance from inserts
|
|
|
|
|
option(DISABLE_STALL_NOTIF "Build with stall notifications" OFF)
|
|
|
|
|
if(DISABLE_STALL_NOTIF)
|
|
|
|
|
add_definitions(-DROCKSDB_DISABLE_STALL_NOTIFICATION)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_DYNAMIC_EXTENSION "build with dynamic extension support" OFF)
|
|
|
|
|
if(NOT WITH_DYNAMIC_EXTENSION)
|
|
|
|
|
add_definitions(-DROCKSDB_NO_DYNAMIC_EXTENSION)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(DEFINED USE_RTTI)
|
|
|
|
|
if(USE_RTTI)
|
|
|
|
|
message(STATUS "Enabling RTTI")
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI")
|
|
|
|
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DROCKSDB_USE_RTTI")
|
|
|
|
|
else()
|
|
|
|
|
if(MSVC)
|
|
|
|
|
message(STATUS "Disabling RTTI in Release builds. Always on in Debug.")
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI")
|
|
|
|
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GR-")
|
|
|
|
|
else()
|
|
|
|
|
message(STATUS "Disabling RTTI in Release builds")
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-rtti")
|
|
|
|
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
else()
|
|
|
|
|
message(STATUS "Enabling RTTI in Debug builds only (default)")
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI")
|
|
|
|
|
if(MSVC)
|
|
|
|
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GR-")
|
|
|
|
|
else()
|
|
|
|
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
# Used to run CI build and tests so we can run faster
|
|
|
|
|
option(OPTDBG "Build optimized debug build with MSVC" OFF)
|
|
|
|
|
option(WITH_RUNTIME_DEBUG "build with debug version of runtime library" ON)
|
|
|
|
|
if(MSVC)
|
|
|
|
|
if(OPTDBG)
|
|
|
|
|
message(STATUS "Debug optimization is enabled")
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "/Oxt")
|
|
|
|
|
else()
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm")
|
|
|
|
|
endif()
|
|
|
|
|
if(WITH_RUNTIME_DEBUG)
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}d")
|
|
|
|
|
else()
|
|
|
|
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}")
|
|
|
|
|
endif()
|
|
|
|
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
|
|
|
|
|
|
|
|
|
|
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
|
|
|
|
|
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(CMAKE_COMPILER_IS_GNUCXX)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-memcmp")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(ROCKSDB_LITE "Build RocksDBLite version" OFF)
|
|
|
|
|
if(ROCKSDB_LITE)
|
|
|
|
|
add_definitions(-DROCKSDB_LITE)
|
|
|
|
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions -Os")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(CMAKE_SYSTEM_NAME MATCHES "Cygwin")
|
|
|
|
|
add_definitions(-fno-builtin-memcmp -DCYGWIN)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
|
|
|
|
add_definitions(-DOS_MACOSX)
|
|
|
|
|
if(CMAKE_SYSTEM_PROCESSOR MATCHES arm)
|
|
|
|
|
add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE)
|
|
|
|
|
# no debug info for IOS, that will make our library big
|
|
|
|
|
add_definitions(-DNDEBUG)
|
|
|
|
|
endif()
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
|
|
|
|
|
add_definitions(-DOS_LINUX)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
|
|
|
|
|
add_definitions(-DOS_SOLARIS)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
|
|
|
|
|
add_definitions(-DOS_FREEBSD)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
|
|
|
|
|
add_definitions(-DOS_NETBSD)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
|
|
|
|
|
add_definitions(-DOS_OPENBSD)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly")
|
|
|
|
|
add_definitions(-DOS_DRAGONFLYBSD)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "Android")
|
|
|
|
|
add_definitions(-DOS_ANDROID)
|
|
|
|
|
elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
|
|
|
|
|
add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
|
|
|
|
|
if(MINGW)
|
|
|
|
|
add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA)
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(NOT WIN32)
|
|
|
|
|
add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_FALLOCATE "build with fallocate" ON)
|
|
|
|
|
if(WITH_FALLOCATE)
|
|
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
|
|
|
|
#include <fcntl.h>
|
|
|
|
|
#include <linux/falloc.h>
|
|
|
|
|
int main() {
|
|
|
|
|
int fd = open(\"/dev/null\", 0);
|
|
|
|
|
fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
|
|
|
|
|
}
|
|
|
|
|
" HAVE_FALLOCATE)
|
|
|
|
|
if(HAVE_FALLOCATE)
|
|
|
|
|
add_definitions(-DROCKSDB_FALLOCATE_PRESENT)
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
|
|
|
|
#include <fcntl.h>
|
|
|
|
|
int main() {
|
|
|
|
|
int fd = open(\"/dev/null\", 0);
|
|
|
|
|
sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
|
|
|
|
|
}
|
|
|
|
|
" HAVE_SYNC_FILE_RANGE_WRITE)
|
|
|
|
|
if(HAVE_SYNC_FILE_RANGE_WRITE)
|
|
|
|
|
add_definitions(-DROCKSDB_RANGESYNC_PRESENT)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
CHECK_CXX_SOURCE_COMPILES("
|
|
|
|
|
#include <pthread.h>
|
|
|
|
|
int main() {
|
|
|
|
|
(void) PTHREAD_MUTEX_ADAPTIVE_NP;
|
|
|
|
|
}
|
|
|
|
|
" HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
|
|
|
|
|
if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
|
|
|
|
|
add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
include(CheckCXXSymbolExists)
|
|
|
|
|
check_cxx_symbol_exists(malloc_usable_size malloc.h HAVE_MALLOC_USABLE_SIZE)
|
|
|
|
|
if(HAVE_MALLOC_USABLE_SIZE)
|
|
|
|
|
add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
check_cxx_symbol_exists(sched_getcpu sched.h HAVE_SCHED_GETCPU)
|
|
|
|
|
if(HAVE_SCHED_GETCPU)
|
|
|
|
|
add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
include_directories(${PROJECT_SOURCE_DIR})
|
|
|
|
|
include_directories(${PROJECT_SOURCE_DIR}/include)
|
|
|
|
|
include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.8.1/fused-src)
|
|
|
|
|
if(WITH_FOLLY_DISTRIBUTED_MUTEX)
|
|
|
|
|
include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
|
|
|
|
|
endif()
|
|
|
|
|
find_package(Threads REQUIRED)
|
|
|
|
|
|
|
|
|
|
# Main library source code
|
|
|
|
|
|
|
|
|
|
set(SOURCES
|
|
|
|
|
cache/clock_cache.cc
|
|
|
|
|
cache/lru_cache.cc
|
|
|
|
|
cache/sharded_cache.cc
|
|
|
|
|
db/arena_wrapped_db_iter.cc
|
|
|
|
|
db/builder.cc
|
|
|
|
|
db/c.cc
|
|
|
|
|
db/column_family.cc
|
|
|
|
|
db/compacted_db_impl.cc
|
|
|
|
|
db/compaction/compaction.cc
|
|
|
|
|
db/compaction/compaction_iterator.cc
|
|
|
|
|
db/compaction/compaction_picker.cc
|
|
|
|
|
db/compaction/compaction_job.cc
|
|
|
|
|
db/compaction/compaction_picker_fifo.cc
|
|
|
|
|
db/compaction/compaction_picker_level.cc
|
|
|
|
|
db/compaction/compaction_picker_universal.cc
|
|
|
|
|
db/convenience.cc
|
|
|
|
|
db/db_filesnapshot.cc
|
|
|
|
|
db/db_impl/db_impl.cc
|
|
|
|
|
db/db_impl/db_impl_write.cc
|
|
|
|
|
db/db_impl/db_impl_compaction_flush.cc
|
|
|
|
|
db/db_impl/db_impl_files.cc
|
|
|
|
|
db/db_impl/db_impl_open.cc
|
|
|
|
|
db/db_impl/db_impl_debug.cc
|
|
|
|
|
db/db_impl/db_impl_experimental.cc
|
|
|
|
|
db/db_impl/db_impl_readonly.cc
|
|
|
|
|
db/db_impl/db_impl_secondary.cc
|
|
|
|
|
db/db_info_dumper.cc
|
|
|
|
|
db/db_iter.cc
|
|
|
|
|
db/dbformat.cc
|
|
|
|
|
db/error_handler.cc
|
|
|
|
|
db/event_helpers.cc
|
|
|
|
|
db/experimental.cc
|
|
|
|
|
db/external_sst_file_ingestion_job.cc
|
|
|
|
|
db/file_indexer.cc
|
|
|
|
|
db/flush_job.cc
|
|
|
|
|
db/flush_scheduler.cc
|
|
|
|
|
db/forward_iterator.cc
|
|
|
|
|
db/import_column_family_job.cc
|
|
|
|
|
db/internal_stats.cc
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
db/logs_with_prep_tracker.cc
|
|
|
|
|
db/log_reader.cc
|
|
|
|
|
db/log_writer.cc
|
|
|
|
|
db/malloc_stats.cc
|
|
|
|
|
db/memtable.cc
|
|
|
|
|
db/memtable_list.cc
|
|
|
|
|
db/merge_helper.cc
|
|
|
|
|
db/merge_operator.cc
|
Compaction Support for Range Deletion
Summary:
This diff introduces RangeDelAggregator, which takes ownership of iterators
provided to it via AddTombstones(). The tombstones are organized in a two-level
map (snapshot stripe -> begin key -> tombstone). Tombstone creation avoids data
copy by holding Slices returned by the iterator, which remain valid thanks to pinning.
For compaction, we create a hierarchical range tombstone iterator with structure
matching the iterator over compaction input data. An aggregator based on that
iterator is used by CompactionIterator to determine which keys are covered by
range tombstones. In case of merge operand, the same aggregator is used by
MergeHelper. Upon finishing each file in the compaction, relevant range tombstones
are added to the output file's range tombstone metablock and file boundaries are
updated accordingly.
To check whether a key is covered by range tombstone, RangeDelAggregator::ShouldDelete()
considers tombstones in the key's snapshot stripe. When this function is used outside of
compaction, it also checks newer stripes, which can contain covering tombstones. Currently
the intra-stripe check involves a linear scan; however, in the future we plan to collapse ranges
within a stripe such that binary search can be used.
RangeDelAggregator::AddToBuilder() adds all range tombstones in the table's key-range
to a new table's range tombstone meta-block. Since range tombstones may fall in the gap
between files, we may need to extend some files' key-ranges. The strategy is (1) first file
extends as far left as possible and other files do not extend left, (2) all files extend right
until either the start of the next file or the end of the last range tombstone in the gap,
whichever comes first.
One other notable change is adding release/move semantics to ScopedArenaIterator
such that it can be used to transfer ownership of an arena-allocated iterator, similar to
how unique_ptr is used for malloc'd data.
Depends on D61473
Test Plan: compaction_iterator_test, mock_table, end-to-end tests in D63927
Reviewers: sdong, IslamAbdelRahman, wanning, yhchiang, lightmark
Reviewed By: lightmark
Subscribers: andrewkr, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D62205
8 years ago
|
|
|
db/range_del_aggregator.cc
|
Use only "local" range tombstones during Get (#4449)
Summary:
Previously, range tombstones were accumulated from every level, which
was necessary if a range tombstone in a higher level covered a key in a lower
level. However, RangeDelAggregator::AddTombstones's complexity is based on
the number of tombstones that are currently stored in it, which is wasteful in
the Get case, where we only need to know the highest sequence number of range
tombstones that cover the key from higher levels, and compute the highest covering
sequence number at the current level. This change introduces this optimization, and
removes the use of RangeDelAggregator from the Get path.
In the benchmark results, the following command was used to initialize the database:
```
./db_bench -db=/dev/shm/5k-rts -use_existing_db=false -benchmarks=filluniquerandom -write_buffer_size=1048576 -compression_type=lz4 -target_file_size_base=1048576 -max_bytes_for_level_base=4194304 -value_size=112 -key_size=16 -block_size=4096 -level_compaction_dynamic_level_bytes=true -num=5000000 -max_background_jobs=12 -benchmark_write_rate_limit=20971520 -range_tombstone_width=100 -writes_per_range_tombstone=100 -max_num_range_tombstones=50000 -bloom_bits=8
```
...and the following command was used to measure read throughput:
```
./db_bench -db=/dev/shm/5k-rts/ -use_existing_db=true -benchmarks=readrandom -disable_auto_compactions=true -num=5000000 -reads=100000 -threads=32
```
The filluniquerandom command was only run once, and the resulting database was used
to measure read performance before and after the PR. Both binaries were compiled with
`DEBUG_LEVEL=0`.
Readrandom results before PR:
```
readrandom : 4.544 micros/op 220090 ops/sec; 16.9 MB/s (63103 of 100000 found)
```
Readrandom results after PR:
```
readrandom : 11.147 micros/op 89707 ops/sec; 6.9 MB/s (63103 of 100000 found)
```
So it's actually slower right now, but this PR paves the way for future optimizations (see #4493).
----
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4449
Differential Revision: D10370575
Pulled By: abhimadan
fbshipit-source-id: 9a2e152be1ef36969055c0e9eb4beb0d96c11f4d
6 years ago
|
|
|
db/range_tombstone_fragmenter.cc
|
|
|
|
|
db/repair.cc
|
|
|
|
|
db/snapshot_impl.cc
|
|
|
|
|
db/table_cache.cc
|
|
|
|
|
db/table_properties_collector.cc
|
|
|
|
|
db/transaction_log_impl.cc
|
Refactor trimming logic for immutable memtables (#5022)
Summary:
MyRocks currently sets `max_write_buffer_number_to_maintain` in order to maintain enough history for transaction conflict checking. The effectiveness of this approach depends on the size of memtables. When memtables are small, it may not keep enough history; when memtables are large, this may consume too much memory.
We are proposing a new way to configure memtable list history: by limiting the memory usage of immutable memtables. The new option is `max_write_buffer_size_to_maintain` and it will take precedence over the old `max_write_buffer_number_to_maintain` if they are both set to non-zero values. The new option accounts for the total memory usage of flushed immutable memtables and mutable memtable. When the total usage exceeds the limit, RocksDB may start dropping immutable memtables (which is also called trimming history), starting from the oldest one.
The semantics of the old option actually works both as an upper bound and lower bound. History trimming will start if number of immutable memtables exceeds the limit, but it will never go below (limit-1) due to history trimming.
In order the mimic the behavior with the new option, history trimming will stop if dropping the next immutable memtable causes the total memory usage go below the size limit. For example, assuming the size limit is set to 64MB, and there are 3 immutable memtables with sizes of 20, 30, 30. Although the total memory usage is 80MB > 64MB, dropping the oldest memtable will reduce the memory usage to 60MB < 64MB, so in this case no memtable will be dropped.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5022
Differential Revision: D14394062
Pulled By: miasantreble
fbshipit-source-id: 60457a509c6af89d0993f988c9b5c2aa9e45f5c5
5 years ago
|
|
|
db/trim_history_scheduler.cc
|
|
|
|
|
db/version_builder.cc
|
|
|
|
|
db/version_edit.cc
|
|
|
|
|
db/version_set.cc
|
|
|
|
|
db/wal_manager.cc
|
|
|
|
|
db/write_batch.cc
|
|
|
|
|
db/write_batch_base.cc
|
|
|
|
|
db/write_controller.cc
|
|
|
|
|
db/write_thread.cc
|
|
|
|
|
env/env.cc
|
|
|
|
|
env/env_chroot.cc
|
|
|
|
|
env/env_encryption.cc
|
|
|
|
|
env/env_hdfs.cc
|
|
|
|
|
env/mock_env.cc
|
|
|
|
|
file/delete_scheduler.cc
|
|
|
|
|
file/file_prefetch_buffer.cc
|
|
|
|
|
file/file_util.cc
|
|
|
|
|
file/filename.cc
|
|
|
|
|
file/random_access_file_reader.cc
|
|
|
|
|
file/read_write_util.cc
|
|
|
|
|
file/readahead_raf.cc
|
|
|
|
|
file/sequence_file_reader.cc
|
|
|
|
|
file/sst_file_manager_impl.cc
|
|
|
|
|
file/writable_file_writer.cc
|
|
|
|
|
logging/auto_roll_logger.cc
|
|
|
|
|
logging/event_logger.cc
|
|
|
|
|
logging/log_buffer.cc
|
|
|
|
|
memory/arena.cc
|
|
|
|
|
memory/concurrent_arena.cc
|
|
|
|
|
memory/jemalloc_nodump_allocator.cc
|
|
|
|
|
memtable/alloc_tracker.cc
|
|
|
|
|
memtable/hash_linklist_rep.cc
|
|
|
|
|
memtable/hash_skiplist_rep.cc
|
|
|
|
|
memtable/skiplistrep.cc
|
|
|
|
|
memtable/vectorrep.cc
|
|
|
|
|
memtable/write_buffer_manager.cc
|
|
|
|
|
monitoring/histogram.cc
|
|
|
|
|
monitoring/histogram_windowing.cc
|
|
|
|
|
monitoring/in_memory_stats_history.cc
|
|
|
|
|
monitoring/instrumented_mutex.cc
|
|
|
|
|
monitoring/iostats_context.cc
|
|
|
|
|
monitoring/perf_context.cc
|
|
|
|
|
monitoring/perf_level.cc
|
|
|
|
|
monitoring/persistent_stats_history.cc
|
|
|
|
|
monitoring/statistics.cc
|
|
|
|
|
monitoring/thread_status_impl.cc
|
|
|
|
|
monitoring/thread_status_updater.cc
|
|
|
|
|
monitoring/thread_status_util.cc
|
|
|
|
|
monitoring/thread_status_util_debug.cc
|
|
|
|
|
options/cf_options.cc
|
|
|
|
|
options/db_options.cc
|
|
|
|
|
options/options.cc
|
|
|
|
|
options/options_helper.cc
|
|
|
|
|
options/options_parser.cc
|
|
|
|
|
options/options_sanity_check.cc
|
|
|
|
|
port/stack_trace.cc
|
|
|
|
|
table/adaptive/adaptive_table_factory.cc
|
|
|
|
|
table/block_based/block.cc
|
|
|
|
|
table/block_based/block_based_filter_block.cc
|
|
|
|
|
table/block_based/block_based_table_builder.cc
|
|
|
|
|
table/block_based/block_based_table_factory.cc
|
|
|
|
|
table/block_based/block_based_table_reader.cc
|
|
|
|
|
table/block_based/block_builder.cc
|
|
|
|
|
table/block_based/block_prefix_index.cc
|
|
|
|
|
table/block_based/data_block_hash_index.cc
|
|
|
|
|
table/block_based/data_block_footer.cc
|
Move the filter readers out of the block cache (#5504)
Summary:
Currently, when the block cache is used for the filter block, it is not
really the block itself that is stored in the cache but a FilterBlockReader
object. Since this object is not pure data (it has, for instance, pointers that
might dangle, including in one case a back pointer to the TableReader), it's not
really sharable. To avoid the issues around this, the current code erases the
cache entries when the TableReader is closed (which, BTW, is not sufficient
since a concurrent TableReader might have picked up the object in the meantime).
Instead of doing this, the patch moves the FilterBlockReader out of the cache
altogether, and decouples the filter reader object from the filter block.
In particular, instead of the TableReader owning, or caching/pinning the
FilterBlockReader (based on the customer's settings), with the change the
TableReader unconditionally owns the FilterBlockReader, which in turn
owns/caches/pins the filter block. This change also enables us to reuse the code
paths historically used for data blocks for filters as well.
Note:
Eviction statistics for filter blocks are temporarily broken. We plan to fix this in a
separate phase.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5504
Test Plan: make asan_check
Differential Revision: D16036974
Pulled By: ltamasi
fbshipit-source-id: 770f543c5fb4ed126fd1e04bfd3809cf4ff9c091
5 years ago
|
|
|
table/block_based/filter_block_reader_common.cc
|
|
|
|
|
table/block_based/flush_block_policy.cc
|
|
|
|
|
table/block_based/full_filter_block.cc
|
|
|
|
|
table/block_based/index_builder.cc
|
|
|
|
|
table/block_based/partitioned_filter_block.cc
|
|
|
|
|
table/block_based/uncompression_dict_reader.cc
|
|
|
|
|
table/block_fetcher.cc
|
|
|
|
|
table/cuckoo/cuckoo_table_builder.cc
|
|
|
|
|
table/cuckoo/cuckoo_table_factory.cc
|
|
|
|
|
table/cuckoo/cuckoo_table_reader.cc
|
|
|
|
|
table/format.cc
|
|
|
|
|
table/get_context.cc
|
|
|
|
|
table/iterator.cc
|
|
|
|
|
table/merging_iterator.cc
|
|
|
|
|
table/meta_blocks.cc
|
|
|
|
|
table/persistent_cache_helper.cc
|
|
|
|
|
table/plain/plain_table_bloom.cc
|
|
|
|
|
table/plain/plain_table_builder.cc
|
|
|
|
|
table/plain/plain_table_factory.cc
|
|
|
|
|
table/plain/plain_table_index.cc
|
|
|
|
|
table/plain/plain_table_key_coding.cc
|
|
|
|
|
table/plain/plain_table_reader.cc
|
|
|
|
|
table/sst_file_reader.cc
|
|
|
|
|
table/sst_file_writer.cc
|
|
|
|
|
table/table_properties.cc
|
|
|
|
|
table/two_level_iterator.cc
|
|
|
|
|
test_util/sync_point.cc
|
|
|
|
|
test_util/sync_point_impl.cc
|
|
|
|
|
test_util/testutil.cc
|
|
|
|
|
test_util/transaction_test_util.cc
|
Block cache simulator: Add pysim to simulate caches using reinforcement learning. (#5610)
Summary:
This PR implements cache eviction using reinforcement learning. It includes two implementations:
1. An implementation of Thompson Sampling for the Bernoulli Bandit [1].
2. An implementation of LinUCB with disjoint linear models [2].
The idea is that a cache uses multiple eviction policies, e.g., MRU, LRU, and LFU. The cache learns which eviction policy is the best and uses it upon a cache miss.
Thompson Sampling is contextless and does not include any features.
LinUCB includes features such as level, block type, caller, column family id to decide which eviction policy to use.
[1] Daniel J. Russo, Benjamin Van Roy, Abbas Kazerouni, Ian Osband, and Zheng Wen. 2018. A Tutorial on Thompson Sampling. Found. Trends Mach. Learn. 11, 1 (July 2018), 1-96. DOI: https://doi.org/10.1561/2200000070
[2] Lihong Li, Wei Chu, John Langford, and Robert E. Schapire. 2010. A contextual-bandit approach to personalized news article recommendation. In Proceedings of the 19th international conference on World wide web (WWW '10). ACM, New York, NY, USA, 661-670. DOI=http://dx.doi.org/10.1145/1772690.1772758
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5610
Differential Revision: D16435067
Pulled By: HaoyuHuang
fbshipit-source-id: 6549239ae14115c01cb1e70548af9e46d8dc21bb
5 years ago
|
|
|
tools/block_cache_analyzer/block_cache_trace_analyzer.cc
|
|
|
|
|
tools/db_bench_tool.cc
|
|
|
|
|
tools/dump/db_dump_tool.cc
|
|
|
|
|
tools/ldb_cmd.cc
|
|
|
|
|
tools/ldb_tool.cc
|
|
|
|
|
tools/sst_dump_tool.cc
|
RocksDB Trace Analyzer (#4091)
Summary:
A framework of trace analyzing for RocksDB
After collecting the trace by using the tool of [PR #3837](https://github.com/facebook/rocksdb/pull/3837). User can use the Trace Analyzer to interpret, analyze, and characterize the collected workload.
**Input:**
1. trace file
2. Whole keys space file
**Statistics:**
1. Access count of each operation (Get, Put, Delete, SingleDelete, DeleteRange, Merge) in each column family.
2. Key hotness (access count) of each one
3. Key space separation based on given prefix
4. Key size distribution
5. Value size distribution if appliable
6. Top K accessed keys
7. QPS statistics including the average QPS and peak QPS
8. Top K accessed prefix
9. The query correlation analyzing, output the number of X after Y and the corresponding average time
intervals
**Output:**
1. key access heat map (either in the accessed key space or whole key space)
2. trace sequence file (interpret the raw trace file to line base text file for future use)
3. Time serial (The key space ID and its access time)
4. Key access count distritbution
5. Key size distribution
6. Value size distribution (in each intervals)
7. whole key space separation by the prefix
8. Accessed key space separation by the prefix
9. QPS of each operation and each column family
10. Top K QPS and their accessed prefix range
**Test:**
1. Added the unit test of analyzing Get, Put, Delete, SingleDelete, DeleteRange, Merge
2. Generated the trace and analyze the trace
**Implemented but not tested (due to the limitation of trace_replay):**
1. Analyzing Iterator, supporting Seek() and SeekForPrev() analyzing
2. Analyzing the number of Key found by Get
**Future Work:**
1. Support execution time analyzing of each requests
2. Support cache hit situation and block read situation of Get
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4091
Differential Revision: D9256157
Pulled By: zhichao-cao
fbshipit-source-id: f0ceacb7eedbc43a3eee6e85b76087d7832a8fe6
6 years ago
|
|
|
tools/trace_analyzer_tool.cc
|
|
|
|
|
trace_replay/trace_replay.cc
|
|
|
|
|
trace_replay/block_cache_tracer.cc
|
|
|
|
|
util/bloom.cc
|
|
|
|
|
util/coding.cc
|
|
|
|
|
util/compaction_job_stats_impl.cc
|
|
|
|
|
util/comparator.cc
|
|
|
|
|
util/compression_context_cache.cc
|
Concurrent task limiter for compaction thread control (#4332)
Summary:
The PR is targeting to resolve the issue of:
https://github.com/facebook/rocksdb/issues/3972#issue-330771918
We have a rocksdb created with leveled-compaction with multiple column families (CFs), some of CFs are using HDD to store big and less frequently accessed data and others are using SSD.
When there are continuously write traffics going on to all CFs, the compaction thread pool is mostly occupied by those slow HDD compactions, which blocks fully utilize SSD bandwidth.
Since atomic write and transaction is needed across CFs, so splitting it to multiple rocksdb instance is not an option for us.
With the compaction thread control, we got 30%+ HDD write throughput gain, and also a lot smooth SSD write since less write stall happening.
ConcurrentTaskLimiter can be shared with multi-CFs across rocksdb instances, so the feature does not only work for multi-CFs scenarios, but also for multi-rocksdbs scenarios, who need disk IO resource control per tenant.
The usage is straight forward:
e.g.:
//
// Enable compaction thread limiter thru ColumnFamilyOptions
//
std::shared_ptr<ConcurrentTaskLimiter> ctl(NewConcurrentTaskLimiter("foo_limiter", 4));
Options options;
ColumnFamilyOptions cf_opt(options);
cf_opt.compaction_thread_limiter = ctl;
...
//
// Compaction thread limiter can be tuned or disabled on-the-fly
//
ctl->SetMaxOutstandingTask(12); // enlarge to 12 tasks
...
ctl->ResetMaxOutstandingTask(); // disable (bypass) thread limiter
ctl->SetMaxOutstandingTask(-1); // Same as above
...
ctl->SetMaxOutstandingTask(0); // full throttle (0 task)
//
// Sharing compaction thread limiter among CFs (to resolve multiple storage perf issue)
//
std::shared_ptr<ConcurrentTaskLimiter> ctl_ssd(NewConcurrentTaskLimiter("ssd_limiter", 8));
std::shared_ptr<ConcurrentTaskLimiter> ctl_hdd(NewConcurrentTaskLimiter("hdd_limiter", 4));
Options options;
ColumnFamilyOptions cf_opt_ssd1(options);
ColumnFamilyOptions cf_opt_ssd2(options);
ColumnFamilyOptions cf_opt_hdd1(options);
ColumnFamilyOptions cf_opt_hdd2(options);
ColumnFamilyOptions cf_opt_hdd3(options);
// SSD CFs
cf_opt_ssd1.compaction_thread_limiter = ctl_ssd;
cf_opt_ssd2.compaction_thread_limiter = ctl_ssd;
// HDD CFs
cf_opt_hdd1.compaction_thread_limiter = ctl_hdd;
cf_opt_hdd2.compaction_thread_limiter = ctl_hdd;
cf_opt_hdd3.compaction_thread_limiter = ctl_hdd;
...
//
// The limiter is disabled by default (or set to nullptr explicitly)
//
Options options;
ColumnFamilyOptions cf_opt(options);
cf_opt.compaction_thread_limiter = nullptr;
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4332
Differential Revision: D13226590
Pulled By: siying
fbshipit-source-id: 14307aec55b8bd59c8223d04aa6db3c03d1b0c1d
6 years ago
|
|
|
util/concurrent_task_limiter_impl.cc
|
|
|
|
|
util/crc32c.cc
|
|
|
|
|
util/dynamic_bloom.cc
|
|
|
|
|
util/filter_policy.cc
|
|
|
|
|
util/hash.cc
|
|
|
|
|
util/murmurhash.cc
|
|
|
|
|
util/random.cc
|
|
|
|
|
util/rate_limiter.cc
|
|
|
|
|
util/slice.cc
|
|
|
|
|
util/status.cc
|
|
|
|
|
util/string_util.cc
|
|
|
|
|
util/thread_local.cc
|
|
|
|
|
util/threadpool_imp.cc
|
|
|
|
|
util/xxhash.cc
|
|
|
|
|
utilities/backupable/backupable_db.cc
|
|
|
|
|
utilities/blob_db/blob_compaction_filter.cc
|
|
|
|
|
utilities/blob_db/blob_db.cc
|
|
|
|
|
utilities/blob_db/blob_db_impl.cc
|
|
|
|
|
utilities/blob_db/blob_db_impl_filesnapshot.cc
|
|
|
|
|
utilities/blob_db/blob_dump_tool.cc
|
|
|
|
|
utilities/blob_db/blob_file.cc
|
|
|
|
|
utilities/blob_db/blob_log_reader.cc
|
|
|
|
|
utilities/blob_db/blob_log_writer.cc
|
|
|
|
|
utilities/blob_db/blob_log_format.cc
|
|
|
|
|
utilities/cassandra/cassandra_compaction_filter.cc
|
|
|
|
|
utilities/cassandra/format.cc
|
|
|
|
|
utilities/cassandra/merge_operator.cc
|
|
|
|
|
utilities/checkpoint/checkpoint_impl.cc
|
|
|
|
|
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
|
|
|
|
|
utilities/debug.cc
|
|
|
|
|
utilities/env_mirror.cc
|
|
|
|
|
utilities/env_timed.cc
|
|
|
|
|
utilities/leveldb_options/leveldb_options.cc
|
|
|
|
|
utilities/memory/memory_util.cc
|
|
|
|
|
utilities/merge_operators/bytesxor.cc
|
|
|
|
|
utilities/merge_operators/max.cc
|
|
|
|
|
utilities/merge_operators/put.cc
|
New API to get all merge operands for a Key (#5604)
Summary:
This is a new API added to db.h to allow for fetching all merge operands associated with a Key. The main motivation for this API is to support use cases where doing a full online merge is not necessary as it is performance sensitive. Example use-cases:
1. Update subset of columns and read subset of columns -
Imagine a SQL Table, a row is encoded as a K/V pair (as it is done in MyRocks). If there are many columns and users only updated one of them, we can use merge operator to reduce write amplification. While users only read one or two columns in the read query, this feature can avoid a full merging of the whole row, and save some CPU.
2. Updating very few attributes in a value which is a JSON-like document -
Updating one attribute can be done efficiently using merge operator, while reading back one attribute can be done more efficiently if we don't need to do a full merge.
----------------------------------------------------------------------------------------------------
API :
Status GetMergeOperands(
const ReadOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* merge_operands,
GetMergeOperandsOptions* get_merge_operands_options,
int* number_of_operands)
Example usage :
int size = 100;
int number_of_operands = 0;
std::vector<PinnableSlice> values(size);
GetMergeOperandsOptions merge_operands_info;
db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k1", values.data(), merge_operands_info, &number_of_operands);
Description :
Returns all the merge operands corresponding to the key. If the number of merge operands in DB is greater than merge_operands_options.expected_max_number_of_operands no merge operands are returned and status is Incomplete. Merge operands returned are in the order of insertion.
merge_operands-> Points to an array of at-least merge_operands_options.expected_max_number_of_operands and the caller is responsible for allocating it. If the status returned is Incomplete then number_of_operands will contain the total number of merge operands found in DB for key.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5604
Test Plan:
Added unit test and perf test in db_bench that can be run using the command:
./db_bench -benchmarks=getmergeoperands --merge_operator=sortlist
Differential Revision: D16657366
Pulled By: vjnadimpalli
fbshipit-source-id: 0faadd752351745224ee12d4ae9ef3cb529951bf
5 years ago
|
|
|
utilities/merge_operators/sortlist.cc
|
|
|
|
|
utilities/merge_operators/string_append/stringappend.cc
|
|
|
|
|
utilities/merge_operators/string_append/stringappend2.cc
|
|
|
|
|
utilities/merge_operators/uint64add.cc
|
|
|
|
|
utilities/object_registry.cc
|
|
|
|
|
utilities/option_change_migration/option_change_migration.cc
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
9 years ago
|
|
|
utilities/options/options_util.cc
|
|
|
|
|
utilities/persistent_cache/block_cache_tier.cc
|
|
|
|
|
utilities/persistent_cache/block_cache_tier_file.cc
|
|
|
|
|
utilities/persistent_cache/block_cache_tier_metadata.cc
|
|
|
|
|
utilities/persistent_cache/persistent_cache_tier.cc
|
|
|
|
|
utilities/persistent_cache/volatile_tier_impl.cc
|
|
|
|
|
utilities/simulator_cache/cache_simulator.cc
|
|
|
|
|
utilities/simulator_cache/sim_cache.cc
|
|
|
|
|
utilities/table_properties_collectors/compact_on_deletion_collector.cc
|
|
|
|
|
utilities/trace/file_trace_reader_writer.cc
|
|
|
|
|
utilities/transactions/optimistic_transaction_db_impl.cc
|
|
|
|
|
utilities/transactions/optimistic_transaction.cc
|
|
|
|
|
utilities/transactions/pessimistic_transaction.cc
|
|
|
|
|
utilities/transactions/pessimistic_transaction_db.cc
|
|
|
|
|
utilities/transactions/snapshot_checker.cc
|
|
|
|
|
utilities/transactions/transaction_base.cc
|
|
|
|
|
utilities/transactions/transaction_db_mutex_impl.cc
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
10 years ago
|
|
|
utilities/transactions/transaction_lock_mgr.cc
|
|
|
|
|
utilities/transactions/transaction_util.cc
|
|
|
|
|
utilities/transactions/write_prepared_txn.cc
|
|
|
|
|
utilities/transactions/write_prepared_txn_db.cc
|
|
|
|
|
utilities/transactions/write_unprepared_txn.cc
|
|
|
|
|
utilities/transactions/write_unprepared_txn_db.cc
|
|
|
|
|
utilities/ttl/db_ttl_impl.cc
|
|
|
|
|
utilities/write_batch_with_index/write_batch_with_index.cc
|
|
|
|
|
utilities/write_batch_with_index/write_batch_with_index_internal.cc
|
|
|
|
|
$<TARGET_OBJECTS:build_version>)
|
|
|
|
|
|
|
|
|
|
if(HAVE_SSE42 AND NOT MSVC)
|
|
|
|
|
set_source_files_properties(
|
|
|
|
|
util/crc32c.cc
|
|
|
|
|
PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(HAVE_POWER8)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
util/crc32c_ppc.c
|
|
|
|
|
util/crc32c_ppc_asm.S)
|
|
|
|
|
endif(HAVE_POWER8)
|
|
|
|
|
|
|
|
|
|
if(HAS_ARMV8_CRC)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
util/crc32c_arm64.cc)
|
|
|
|
|
endif(HAS_ARMV8_CRC)
|
|
|
|
|
|
|
|
|
|
if(WIN32)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
port/win/io_win.cc
|
|
|
|
|
port/win/env_win.cc
|
|
|
|
|
port/win/env_default.cc
|
|
|
|
|
port/win/port_win.cc
|
|
|
|
|
port/win/win_logger.cc
|
|
|
|
|
port/win/win_thread.cc)
|
|
|
|
|
|
|
|
|
|
if(WITH_XPRESS)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
port/win/xpress_win.cc)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(WITH_JEMALLOC)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
port/win/win_jemalloc.cc)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
else()
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
port/port_posix.cc
|
|
|
|
|
env/env_posix.cc
|
|
|
|
|
env/io_posix.cc)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(WITH_FOLLY_DISTRIBUTED_MUTEX)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
third-party/folly/folly/detail/Futex.cpp
|
|
|
|
|
third-party/folly/folly/synchronization/AtomicNotification.cpp
|
|
|
|
|
third-party/folly/folly/synchronization/DistributedMutex.cpp
|
|
|
|
|
third-party/folly/folly/synchronization/ParkingLot.cpp
|
|
|
|
|
third-party/folly/folly/synchronization/WaitOptions.cpp)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
|
|
|
|
|
set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
|
|
|
|
|
set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
|
|
|
|
|
|
|
|
|
|
option(WITH_LIBRADOS "Build with librados" OFF)
|
|
|
|
|
if(WITH_LIBRADOS)
|
|
|
|
|
list(APPEND SOURCES
|
|
|
|
|
utilities/env_librados.cc)
|
|
|
|
|
list(APPEND THIRDPARTY_LIBS rados)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(WIN32)
|
|
|
|
|
set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
|
|
|
|
|
set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
else()
|
cross-platform compatibility improvements
Summary:
We've had a couple CockroachDB users fail to build RocksDB on exotic platforms, so I figured I'd try my hand at solving these issues upstream. The problems stem from a) `USE_SSE=1` being too aggressive about turning on SSE4.2, even on toolchains that don't support SSE4.2 and b) RocksDB attempting to detect support for thread-local storage based on OS, even though it can vary by compiler on the same OS.
See the individual commit messages for details. Regarding SSE support, this PR should change virtually nothing for non-CMake based builds. `make`, `PORTABLE=1 make`, `USE_SSE=1 make`, and `PORTABLE=1 USE_SSE=1 make` function exactly as before, except that SSE support will be automatically disabled when a simple SSE4.2-using test program fails to compile, as it does on OpenBSD. (OpenBSD's ports GCC supports SSE4.2, but its binutils do not, so `__SSE_4_2__` is defined but an SSE4.2-using program will fail to assemble.) A warning is emitted in this case. The CMake build is modified to support the same set of options, except that `USE_SSE` is spelled `FORCE_SSE42` because `USE_SSE` is rather useless now that we can automatically detect SSE support, and I figure changing options in the CMake build is less disruptive than changing the non-CMake build.
I've tested these changes on all the platforms I can get my hands on (macOS, Windows MSVC, Windows MinGW, and OpenBSD) and it all works splendidly. Let me know if there's anything you object to—I obviously don't mean to break any of your build pipelines in the process of fixing ours downstream.
Closes https://github.com/facebook/rocksdb/pull/2199
Differential Revision: D5054042
Pulled By: yiwu-arbug
fbshipit-source-id: 938e1fc665c049c02ae15698e1409155b8e72171
8 years ago
|
|
|
set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
|
|
|
|
|
set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
|
|
|
|
|
add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
|
|
|
|
|
target_link_libraries(${ROCKSDB_SHARED_LIB}
|
|
|
|
|
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
|
|
|
|
|
LINKER_LANGUAGE CXX
|
|
|
|
|
VERSION ${rocksdb_VERSION}
|
|
|
|
|
SOVERSION ${rocksdb_VERSION_MAJOR}
|
|
|
|
|
CXX_STANDARD 11
|
|
|
|
|
OUTPUT_NAME "rocksdb")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
|
|
|
|
|
target_link_libraries(${ROCKSDB_STATIC_LIB}
|
|
|
|
|
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
|
|
|
|
|
if(WIN32)
|
|
|
|
|
add_library(${ROCKSDB_IMPORT_LIB} SHARED ${SOURCES})
|
|
|
|
|
target_link_libraries(${ROCKSDB_IMPORT_LIB}
|
|
|
|
|
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
|
|
|
|
set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
|
|
|
|
|
COMPILE_DEFINITIONS "ROCKSDB_DLL;ROCKSDB_LIBRARY_EXPORTS")
|
|
|
|
|
if(MSVC)
|
|
|
|
|
set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES
|
|
|
|
|
COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_STATIC_LIB}.pdb")
|
|
|
|
|
set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
|
|
|
|
|
COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_IMPORT_LIB}.pdb")
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_JNI "build with JNI" OFF)
|
|
|
|
|
if(WITH_JNI OR JNI)
|
|
|
|
|
message(STATUS "JNI library is enabled")
|
|
|
|
|
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
|
|
|
|
|
else()
|
|
|
|
|
message(STATUS "JNI library is disabled")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
# Installation and packaging
|
|
|
|
|
if(WIN32)
|
|
|
|
|
option(ROCKSDB_INSTALL_ON_WINDOWS "Enable install target on Windows" OFF)
|
|
|
|
|
endif()
|
|
|
|
|
if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
|
|
|
|
|
if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
|
|
|
|
|
if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
|
|
|
|
|
# Change default installation prefix on Linux to /usr
|
|
|
|
|
set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "Install path prefix, prepended onto install directories." FORCE)
|
|
|
|
|
endif()
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
include(GNUInstallDirs)
|
|
|
|
|
include(CMakePackageConfigHelpers)
|
|
|
|
|
|
|
|
|
|
set(package_config_destination ${CMAKE_INSTALL_LIBDIR}/cmake/rocksdb)
|
|
|
|
|
|
|
|
|
|
configure_package_config_file(
|
|
|
|
|
${CMAKE_CURRENT_LIST_DIR}/cmake/RocksDBConfig.cmake.in RocksDBConfig.cmake
|
|
|
|
|
INSTALL_DESTINATION ${package_config_destination}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
write_basic_package_version_file(
|
|
|
|
|
RocksDBConfigVersion.cmake
|
|
|
|
|
VERSION ${rocksdb_VERSION}
|
|
|
|
|
COMPATIBILITY SameMajorVersion
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
|
|
|
|
|
|
|
|
|
|
install(
|
|
|
|
|
TARGETS ${ROCKSDB_STATIC_LIB}
|
|
|
|
|
EXPORT RocksDBTargets
|
|
|
|
|
COMPONENT devel
|
|
|
|
|
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
|
|
|
|
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
install(
|
|
|
|
|
TARGETS ${ROCKSDB_SHARED_LIB}
|
|
|
|
|
EXPORT RocksDBTargets
|
|
|
|
|
COMPONENT runtime
|
|
|
|
|
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
|
|
|
|
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
|
|
|
|
|
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
|
|
|
|
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
install(
|
|
|
|
|
EXPORT RocksDBTargets
|
|
|
|
|
COMPONENT devel
|
|
|
|
|
DESTINATION ${package_config_destination}
|
|
|
|
|
NAMESPACE RocksDB::
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
install(
|
|
|
|
|
FILES
|
|
|
|
|
${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfig.cmake
|
|
|
|
|
${CMAKE_CURRENT_BINARY_DIR}/RocksDBConfigVersion.cmake
|
|
|
|
|
COMPONENT devel
|
|
|
|
|
DESTINATION ${package_config_destination}
|
|
|
|
|
)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_TESTS "build with tests" ON)
|
|
|
|
|
if(WITH_TESTS)
|
|
|
|
|
add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest)
|
|
|
|
|
set(TESTS
|
|
|
|
|
cache/cache_test.cc
|
|
|
|
|
cache/lru_cache_test.cc
|
|
|
|
|
db/column_family_test.cc
|
|
|
|
|
db/compact_files_test.cc
|
|
|
|
|
db/compaction/compaction_job_stats_test.cc
|
|
|
|
|
db/compaction/compaction_job_test.cc
|
|
|
|
|
db/compaction/compaction_iterator_test.cc
|
|
|
|
|
db/compaction/compaction_picker_test.cc
|
|
|
|
|
db/comparator_db_test.cc
|
|
|
|
|
db/corruption_test.cc
|
|
|
|
|
db/cuckoo_table_db_test.cc
|
|
|
|
|
db/db_basic_test.cc
|
|
|
|
|
db/db_blob_index_test.cc
|
|
|
|
|
db/db_block_cache_test.cc
|
|
|
|
|
db/db_bloom_filter_test.cc
|
|
|
|
|
db/db_compaction_filter_test.cc
|
|
|
|
|
db/db_compaction_test.cc
|
|
|
|
|
db/db_dynamic_level_test.cc
|
|
|
|
|
db/db_flush_test.cc
|
|
|
|
|
db/db_inplace_update_test.cc
|
|
|
|
|
db/db_io_failure_test.cc
|
|
|
|
|
db/db_iter_test.cc
|
Change and clarify the relationship between Valid(), status() and Seek*() for all iterators. Also fix some bugs
Summary:
Before this PR, Iterator/InternalIterator may simultaneously have non-ok status() and Valid() = true. That state means that the last operation failed, but the iterator is nevertheless positioned on some unspecified record. Likely intended uses of that are:
* If some sst files are corrupted, a normal iterator can be used to read the data from files that are not corrupted.
* When using read_tier = kBlockCacheTier, read the data that's in block cache, skipping over the data that is not.
However, this behavior wasn't documented well (and until recently the wiki on github had misleading incorrect information). In the code there's a lot of confusion about the relationship between status() and Valid(), and about whether Seek()/SeekToLast()/etc reset the status or not. There were a number of bugs caused by this confusion, both inside rocksdb and in the code that uses rocksdb (including ours).
This PR changes the convention to:
* If status() is not ok, Valid() always returns false.
* Any seek operation resets status. (Before the PR, it depended on iterator type and on particular error.)
This does sacrifice the two use cases listed above, but siying said it's ok.
Overview of the changes:
* A commit that adds missing status checks in MergingIterator. This fixes a bug that actually affects us, and we need it fixed. `DBIteratorTest.NonBlockingIterationBugRepro` explains the scenario.
* Changes to lots of iterator types to make all of them conform to the new convention. Some bug fixes along the way. By far the biggest changes are in DBIter, which is a big messy piece of code; I tried to make it less big and messy but mostly failed.
* A stress-test for DBIter, to gain some confidence that I didn't break it. It does a few million random operations on the iterator, while occasionally modifying the underlying data (like ForwardIterator does) and occasionally returning non-ok status from internal iterator.
To find the iterator types that needed changes I searched for "public .*Iterator" in the code. Here's an overview of all 27 iterator types:
Iterators that didn't need changes:
* status() is always ok(), or Valid() is always false: MemTableIterator, ModelIter, TestIterator, KVIter (2 classes with this name anonymous namespaces), LoggingForwardVectorIterator, VectorIterator, MockTableIterator, EmptyIterator, EmptyInternalIterator.
* Thin wrappers that always pass through Valid() and status(): ArenaWrappedDBIter, TtlIterator, InternalIteratorFromIterator.
Iterators with changes (see inline comments for details):
* DBIter - an overhaul:
- It used to silently skip corrupted keys (`FindParseableKey()`), which seems dangerous. This PR makes it just stop immediately after encountering a corrupted key, just like it would for other kinds of corruption. Let me know if there was actually some deeper meaning in this behavior and I should put it back.
- It had a few code paths silently discarding subiterator's status. The stress test caught a few.
- The backwards iteration code path was expecting the internal iterator's set of keys to be immutable. It's probably always true in practice at the moment, since ForwardIterator doesn't support backwards iteration, but this PR fixes it anyway. See added DBIteratorTest.ReverseToForwardBug for an example.
- Some parts of backwards iteration code path even did things like `assert(iter_->Valid())` after a seek, which is never a safe assumption.
- It used to not reset status on seek for some types of errors.
- Some simplifications and better comments.
- Some things got more complicated from the added error handling. I'm open to ideas for how to make it nicer.
* MergingIterator - check status after every operation on every subiterator, and in some places assert that valid subiterators have ok status.
* ForwardIterator - changed to the new convention, also slightly simplified.
* ForwardLevelIterator - fixed some bugs and simplified.
* LevelIterator - simplified.
* TwoLevelIterator - changed to the new convention. Also fixed a bug that would make SeekForPrev() sometimes silently ignore errors from first_level_iter_.
* BlockBasedTableIterator - minor changes.
* BlockIter - replaced `SetStatus()` with `Invalidate()` to make sure non-ok BlockIter is always invalid.
* PlainTableIterator - some seeks used to not reset status.
* CuckooTableIterator - tiny code cleanup.
* ManagedIterator - fixed some bugs.
* BaseDeltaIterator - changed to the new convention and fixed a bug.
* BlobDBIterator - seeks used to not reset status.
* KeyConvertingIterator - some small change.
Closes https://github.com/facebook/rocksdb/pull/3810
Differential Revision: D7888019
Pulled By: al13n321
fbshipit-source-id: 4aaf6d3421c545d16722a815b2fa2e7912bc851d
7 years ago
|
|
|
db/db_iter_stress_test.cc
|
|
|
|
|
db/db_iterator_test.cc
|
|
|
|
|
db/db_log_iter_test.cc
|
|
|
|
|
db/db_memtable_test.cc
|
|
|
|
|
db/db_merge_operator_test.cc
|
New API to get all merge operands for a Key (#5604)
Summary:
This is a new API added to db.h to allow for fetching all merge operands associated with a Key. The main motivation for this API is to support use cases where doing a full online merge is not necessary as it is performance sensitive. Example use-cases:
1. Update subset of columns and read subset of columns -
Imagine a SQL Table, a row is encoded as a K/V pair (as it is done in MyRocks). If there are many columns and users only updated one of them, we can use merge operator to reduce write amplification. While users only read one or two columns in the read query, this feature can avoid a full merging of the whole row, and save some CPU.
2. Updating very few attributes in a value which is a JSON-like document -
Updating one attribute can be done efficiently using merge operator, while reading back one attribute can be done more efficiently if we don't need to do a full merge.
----------------------------------------------------------------------------------------------------
API :
Status GetMergeOperands(
const ReadOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* merge_operands,
GetMergeOperandsOptions* get_merge_operands_options,
int* number_of_operands)
Example usage :
int size = 100;
int number_of_operands = 0;
std::vector<PinnableSlice> values(size);
GetMergeOperandsOptions merge_operands_info;
db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(), "k1", values.data(), merge_operands_info, &number_of_operands);
Description :
Returns all the merge operands corresponding to the key. If the number of merge operands in DB is greater than merge_operands_options.expected_max_number_of_operands no merge operands are returned and status is Incomplete. Merge operands returned are in the order of insertion.
merge_operands-> Points to an array of at-least merge_operands_options.expected_max_number_of_operands and the caller is responsible for allocating it. If the status returned is Incomplete then number_of_operands will contain the total number of merge operands found in DB for key.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5604
Test Plan:
Added unit test and perf test in db_bench that can be run using the command:
./db_bench -benchmarks=getmergeoperands --merge_operator=sortlist
Differential Revision: D16657366
Pulled By: vjnadimpalli
fbshipit-source-id: 0faadd752351745224ee12d4ae9ef3cb529951bf
5 years ago
|
|
|
db/db_merge_operand_test.cc
|
|
|
|
|
db/db_options_test.cc
|
|
|
|
|
db/db_properties_test.cc
|
|
|
|
|
db/db_range_del_test.cc
|
|
|
|
|
db/db_impl/db_secondary_test.cc
|
|
|
|
|
db/db_sst_test.cc
|
|
|
|
|
db/db_statistics_test.cc
|
|
|
|
|
db/db_table_properties_test.cc
|
|
|
|
|
db/db_tailing_iter_test.cc
|
|
|
|
|
db/db_test.cc
|
|
|
|
|
db/db_test2.cc
|
|
|
|
|
db/db_universal_compaction_test.cc
|
|
|
|
|
db/db_wal_test.cc
|
|
|
|
|
db/db_write_test.cc
|
|
|
|
|
db/dbformat_test.cc
|
|
|
|
|
db/deletefile_test.cc
|
|
|
|
|
db/error_handler_test.cc
|
|
|
|
|
db/obsolete_files_test.cc
|
|
|
|
|
db/external_sst_file_basic_test.cc
|
|
|
|
|
db/external_sst_file_test.cc
|
|
|
|
|
db/fault_injection_test.cc
|
|
|
|
|
db/file_indexer_test.cc
|
|
|
|
|
db/filename_test.cc
|
|
|
|
|
db/flush_job_test.cc
|
|
|
|
|
db/listener_test.cc
|
|
|
|
|
db/log_test.cc
|
|
|
|
|
db/manual_compaction_test.cc
|
|
|
|
|
db/memtable_list_test.cc
|
|
|
|
|
db/merge_helper_test.cc
|
|
|
|
|
db/merge_test.cc
|
|
|
|
|
db/options_file_test.cc
|
|
|
|
|
db/perf_context_test.cc
|
|
|
|
|
db/plain_table_db_test.cc
|
|
|
|
|
db/prefix_test.cc
|
Use only "local" range tombstones during Get (#4449)
Summary:
Previously, range tombstones were accumulated from every level, which
was necessary if a range tombstone in a higher level covered a key in a lower
level. However, RangeDelAggregator::AddTombstones's complexity is based on
the number of tombstones that are currently stored in it, which is wasteful in
the Get case, where we only need to know the highest sequence number of range
tombstones that cover the key from higher levels, and compute the highest covering
sequence number at the current level. This change introduces this optimization, and
removes the use of RangeDelAggregator from the Get path.
In the benchmark results, the following command was used to initialize the database:
```
./db_bench -db=/dev/shm/5k-rts -use_existing_db=false -benchmarks=filluniquerandom -write_buffer_size=1048576 -compression_type=lz4 -target_file_size_base=1048576 -max_bytes_for_level_base=4194304 -value_size=112 -key_size=16 -block_size=4096 -level_compaction_dynamic_level_bytes=true -num=5000000 -max_background_jobs=12 -benchmark_write_rate_limit=20971520 -range_tombstone_width=100 -writes_per_range_tombstone=100 -max_num_range_tombstones=50000 -bloom_bits=8
```
...and the following command was used to measure read throughput:
```
./db_bench -db=/dev/shm/5k-rts/ -use_existing_db=true -benchmarks=readrandom -disable_auto_compactions=true -num=5000000 -reads=100000 -threads=32
```
The filluniquerandom command was only run once, and the resulting database was used
to measure read performance before and after the PR. Both binaries were compiled with
`DEBUG_LEVEL=0`.
Readrandom results before PR:
```
readrandom : 4.544 micros/op 220090 ops/sec; 16.9 MB/s (63103 of 100000 found)
```
Readrandom results after PR:
```
readrandom : 11.147 micros/op 89707 ops/sec; 6.9 MB/s (63103 of 100000 found)
```
So it's actually slower right now, but this PR paves the way for future optimizations (see #4493).
----
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4449
Differential Revision: D10370575
Pulled By: abhimadan
fbshipit-source-id: 9a2e152be1ef36969055c0e9eb4beb0d96c11f4d
6 years ago
|
|
|
db/range_del_aggregator_test.cc
|
|
|
|
|
db/range_tombstone_fragmenter_test.cc
|
|
|
|
|
db/repair_test.cc
|
|
|
|
|
db/table_properties_collector_test.cc
|
|
|
|
|
db/version_builder_test.cc
|
|
|
|
|
db/version_edit_test.cc
|
|
|
|
|
db/version_set_test.cc
|
|
|
|
|
db/wal_manager_test.cc
|
|
|
|
|
db/write_batch_test.cc
|
|
|
|
|
db/write_callback_test.cc
|
|
|
|
|
db/write_controller_test.cc
|
|
|
|
|
env/env_basic_test.cc
|
|
|
|
|
env/env_test.cc
|
|
|
|
|
env/mock_env_test.cc
|
|
|
|
|
file/delete_scheduler_test.cc
|
|
|
|
|
logging/auto_roll_logger_test.cc
|
|
|
|
|
logging/env_logger_test.cc
|
|
|
|
|
logging/event_logger_test.cc
|
|
|
|
|
memory/arena_test.cc
|
|
|
|
|
memtable/inlineskiplist_test.cc
|
|
|
|
|
memtable/skiplist_test.cc
|
|
|
|
|
memtable/write_buffer_manager_test.cc
|
|
|
|
|
monitoring/histogram_test.cc
|
|
|
|
|
monitoring/iostats_context_test.cc
|
|
|
|
|
monitoring/statistics_test.cc
|
|
|
|
|
monitoring/stats_history_test.cc
|
|
|
|
|
options/options_settable_test.cc
|
|
|
|
|
options/options_test.cc
|
|
|
|
|
table/block_based/block_based_filter_block_test.cc
|
|
|
|
|
table/block_based/block_test.cc
|
|
|
|
|
table/block_based/data_block_hash_index_test.cc
|
|
|
|
|
table/block_based/full_filter_block_test.cc
|
|
|
|
|
table/block_based/partitioned_filter_block_test.cc
|
|
|
|
|
table/cleanable_test.cc
|
|
|
|
|
table/cuckoo/cuckoo_table_builder_test.cc
|
|
|
|
|
table/cuckoo/cuckoo_table_reader_test.cc
|
|
|
|
|
table/merger_test.cc
|
|
|
|
|
table/sst_file_reader_test.cc
|
|
|
|
|
table/table_test.cc
|
Block cache simulator: Add pysim to simulate caches using reinforcement learning. (#5610)
Summary:
This PR implements cache eviction using reinforcement learning. It includes two implementations:
1. An implementation of Thompson Sampling for the Bernoulli Bandit [1].
2. An implementation of LinUCB with disjoint linear models [2].
The idea is that a cache uses multiple eviction policies, e.g., MRU, LRU, and LFU. The cache learns which eviction policy is the best and uses it upon a cache miss.
Thompson Sampling is contextless and does not include any features.
LinUCB includes features such as level, block type, caller, column family id to decide which eviction policy to use.
[1] Daniel J. Russo, Benjamin Van Roy, Abbas Kazerouni, Ian Osband, and Zheng Wen. 2018. A Tutorial on Thompson Sampling. Found. Trends Mach. Learn. 11, 1 (July 2018), 1-96. DOI: https://doi.org/10.1561/2200000070
[2] Lihong Li, Wei Chu, John Langford, and Robert E. Schapire. 2010. A contextual-bandit approach to personalized news article recommendation. In Proceedings of the 19th international conference on World wide web (WWW '10). ACM, New York, NY, USA, 661-670. DOI=http://dx.doi.org/10.1145/1772690.1772758
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5610
Differential Revision: D16435067
Pulled By: HaoyuHuang
fbshipit-source-id: 6549239ae14115c01cb1e70548af9e46d8dc21bb
5 years ago
|
|
|
tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc
|
|
|
|
|
tools/ldb_cmd_test.cc
|
|
|
|
|
tools/reduce_levels_test.cc
|
|
|
|
|
tools/sst_dump_test.cc
|
RocksDB Trace Analyzer (#4091)
Summary:
A framework of trace analyzing for RocksDB
After collecting the trace by using the tool of [PR #3837](https://github.com/facebook/rocksdb/pull/3837). User can use the Trace Analyzer to interpret, analyze, and characterize the collected workload.
**Input:**
1. trace file
2. Whole keys space file
**Statistics:**
1. Access count of each operation (Get, Put, Delete, SingleDelete, DeleteRange, Merge) in each column family.
2. Key hotness (access count) of each one
3. Key space separation based on given prefix
4. Key size distribution
5. Value size distribution if appliable
6. Top K accessed keys
7. QPS statistics including the average QPS and peak QPS
8. Top K accessed prefix
9. The query correlation analyzing, output the number of X after Y and the corresponding average time
intervals
**Output:**
1. key access heat map (either in the accessed key space or whole key space)
2. trace sequence file (interpret the raw trace file to line base text file for future use)
3. Time serial (The key space ID and its access time)
4. Key access count distritbution
5. Key size distribution
6. Value size distribution (in each intervals)
7. whole key space separation by the prefix
8. Accessed key space separation by the prefix
9. QPS of each operation and each column family
10. Top K QPS and their accessed prefix range
**Test:**
1. Added the unit test of analyzing Get, Put, Delete, SingleDelete, DeleteRange, Merge
2. Generated the trace and analyze the trace
**Implemented but not tested (due to the limitation of trace_replay):**
1. Analyzing Iterator, supporting Seek() and SeekForPrev() analyzing
2. Analyzing the number of Key found by Get
**Future Work:**
1. Support execution time analyzing of each requests
2. Support cache hit situation and block read situation of Get
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4091
Differential Revision: D9256157
Pulled By: zhichao-cao
fbshipit-source-id: f0ceacb7eedbc43a3eee6e85b76087d7832a8fe6
6 years ago
|
|
|
tools/trace_analyzer_test.cc
|
|
|
|
|
util/autovector_test.cc
|
|
|
|
|
util/bloom_test.cc
|
|
|
|
|
util/coding_test.cc
|
|
|
|
|
util/crc32c_test.cc
|
|
|
|
|
util/dynamic_bloom_test.cc
|
RangeSync not to sync last 1MB of the file
Summary:
From other ones' investigation:
"sync_file_range() behavior highly depends on kernel version and filesystem.
xfs does neighbor page flushing outside of the specified ranges. For example, sync_file_range(fd, 8192, 16384) does not only trigger flushing page #3 to #4, but also flushing many more dirty pages (i.e. up to page#16)... Ranges of the sync_file_range() should be far enough from write() offset (at least 1MB)."
Test Plan: make all check
Reviewers: igor, rven, kradhakrishnan, yhchiang, IslamAbdelRahman, anthony
Reviewed By: anthony
Subscribers: yoshinorim, MarkCallaghan, sumeet, domas, dhruba, leveldb, ljin
Differential Revision: https://reviews.facebook.net/D15807
9 years ago
|
|
|
util/file_reader_writer_test.cc
|
|
|
|
|
util/filelock_test.cc
|
|
|
|
|
util/hash_test.cc
|
|
|
|
|
util/heap_test.cc
|
|
|
|
|
util/rate_limiter_test.cc
|
|
|
|
|
util/repeatable_thread_test.cc
|
|
|
|
|
util/slice_transform_test.cc
|
|
|
|
|
util/timer_queue_test.cc
|
|
|
|
|
util/thread_list_test.cc
|
|
|
|
|
util/thread_local_test.cc
|
|
|
|
|
utilities/backupable/backupable_db_test.cc
|
|
|
|
|
utilities/blob_db/blob_db_test.cc
|
|
|
|
|
utilities/cassandra/cassandra_functional_test.cc
|
|
|
|
|
utilities/cassandra/cassandra_format_test.cc
|
|
|
|
|
utilities/cassandra/cassandra_row_merge_test.cc
|
|
|
|
|
utilities/cassandra/cassandra_serialize_test.cc
|
|
|
|
|
utilities/checkpoint/checkpoint_test.cc
|
|
|
|
|
utilities/memory/memory_test.cc
|
|
|
|
|
utilities/merge_operators/string_append/stringappend_test.cc
|
|
|
|
|
utilities/object_registry_test.cc
|
|
|
|
|
utilities/option_change_migration/option_change_migration_test.cc
|
|
|
|
|
utilities/options/options_util_test.cc
|
|
|
|
|
utilities/persistent_cache/hash_table_test.cc
|
|
|
|
|
utilities/persistent_cache/persistent_cache_test.cc
|
|
|
|
|
utilities/simulator_cache/cache_simulator_test.cc
|
|
|
|
|
utilities/simulator_cache/sim_cache_test.cc
|
|
|
|
|
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
|
|
|
|
|
utilities/transactions/optimistic_transaction_test.cc
|
Pessimistic Transactions
Summary:
Initial implementation of Pessimistic Transactions. This diff contains the api changes discussed in D38913. This diff is pretty large, so let me know if people would prefer to meet up to discuss it.
MyRocks folks: please take a look at the API in include/rocksdb/utilities/transaction[_db].h and let me know if you have any issues.
Also, you'll notice a couple of TODOs in the implementation of RollbackToSavePoint(). After chatting with Siying, I'm going to send out a separate diff for an alternate implementation of this feature that implements the rollback inside of WriteBatch/WriteBatchWithIndex. We can then decide which route is preferable.
Next, I'm planning on doing some perf testing and then integrating this diff into MongoRocks for further testing.
Test Plan: Unit tests, db_bench parallel testing.
Reviewers: igor, rven, sdong, yhchiang, yoshinorim
Reviewed By: sdong
Subscribers: hermanlee4, maykov, spetrunia, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D40869
10 years ago
|
|
|
utilities/transactions/transaction_test.cc
|
|
|
|
|
utilities/transactions/write_prepared_transaction_test.cc
|
|
|
|
|
utilities/transactions/write_unprepared_transaction_test.cc
|
|
|
|
|
utilities/ttl/ttl_test.cc
|
|
|
|
|
utilities/write_batch_with_index/write_batch_with_index_test.cc
|
|
|
|
|
)
|
|
|
|
|
if(WITH_LIBRADOS)
|
|
|
|
|
list(APPEND TESTS utilities/env_librados_test.cc)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
if(WITH_FOLLY_DISTRIBUTED_MUTEX)
|
|
|
|
|
list(APPEND TESTS third-party/folly/folly/synchronization/test/DistributedMutexTest.cpp)
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
set(BENCHMARKS
|
|
|
|
|
cache/cache_bench.cc
|
|
|
|
|
memtable/memtablerep_bench.cc
|
|
|
|
|
db/range_del_aggregator_bench.cc
|
|
|
|
|
tools/db_bench.cc
|
|
|
|
|
table/table_reader_bench.cc
|
|
|
|
|
utilities/persistent_cache/hash_table_bench.cc)
|
|
|
|
|
add_library(testharness OBJECT test_util/testharness.cc)
|
|
|
|
|
foreach(sourcefile ${BENCHMARKS})
|
|
|
|
|
get_filename_component(exename ${sourcefile} NAME_WE)
|
|
|
|
|
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
|
|
|
|
|
$<TARGET_OBJECTS:testharness>)
|
|
|
|
|
target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
|
|
|
|
|
endforeach(sourcefile ${BENCHMARKS})
|
|
|
|
|
|
|
|
|
|
# For test util library that is build only in DEBUG mode
|
|
|
|
|
# and linked to tests. Add test only code that is not #ifdefed for Release here.
|
|
|
|
|
set(TESTUTIL_SOURCE
|
|
|
|
|
db/db_test_util.cc
|
|
|
|
|
monitoring/thread_status_updater_debug.cc
|
|
|
|
|
table/mock_table.cc
|
|
|
|
|
test_util/fault_injection_test_env.cc
|
|
|
|
|
utilities/cassandra/test_utils.cc
|
|
|
|
|
)
|
|
|
|
|
# test utilities are only build in debug
|
|
|
|
|
enable_testing()
|
|
|
|
|
add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
|
|
|
|
|
set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
|
|
|
|
|
add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
|
|
|
|
|
if(MSVC)
|
|
|
|
|
set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
|
|
|
|
|
endif()
|
|
|
|
|
set_target_properties(${TESTUTILLIB}
|
|
|
|
|
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# Tests are excluded from Release builds
|
|
|
|
|
set(TEST_EXES ${TESTS})
|
|
|
|
|
|
|
|
|
|
foreach(sourcefile ${TEST_EXES})
|
|
|
|
|
get_filename_component(exename ${sourcefile} NAME_WE)
|
|
|
|
|
add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile}
|
|
|
|
|
$<TARGET_OBJECTS:testharness>)
|
|
|
|
|
set_target_properties(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX}
|
|
|
|
|
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
|
|
|
|
|
OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
|
|
|
|
|
)
|
|
|
|
|
target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
|
|
|
|
|
if(NOT "${exename}" MATCHES "db_sanity_test")
|
|
|
|
|
add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
|
|
|
|
|
add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
|
|
|
|
|
endif()
|
|
|
|
|
endforeach(sourcefile ${TEST_EXES})
|
|
|
|
|
|
|
|
|
|
# C executables must link to a shared object
|
|
|
|
|
set(C_TESTS db/c_test.c)
|
|
|
|
|
set(C_TEST_EXES ${C_TESTS})
|
|
|
|
|
|
|
|
|
|
foreach(sourcefile ${C_TEST_EXES})
|
|
|
|
|
string(REPLACE ".c" "" exename ${sourcefile})
|
|
|
|
|
string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
|
|
|
|
|
add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
|
|
|
|
|
set_target_properties(${exename}${ARTIFACT_SUFFIX}
|
|
|
|
|
PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
|
|
|
|
|
EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
|
|
|
|
|
)
|
|
|
|
|
target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
|
|
|
|
|
add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
|
|
|
|
|
add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
|
|
|
|
|
endforeach(sourcefile ${C_TEST_EXES})
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
option(WITH_TOOLS "build with tools" ON)
|
|
|
|
|
if(WITH_TOOLS)
|
|
|
|
|
add_subdirectory(tools)
|
|
|
|
|
endif()
|
