Wei Wei
10 years ago
commit
d1cafc0892
@ -0,0 +1,11 @@ |
||||
Facebook Inc. |
||||
Facebook Engineering Team |
||||
|
||||
Google Inc. |
||||
# Initial version authors: |
||||
Jeffrey Dean <jeff@google.com> |
||||
Sanjay Ghemawat <sanjay@google.com> |
||||
|
||||
# Partial list of contributors: |
||||
Kevin Regan <kevin.d.regan@gmail.com> |
||||
Johan Bilien <jobi@litl.com> |
||||
@ -0,0 +1,16 @@ |
||||
Vagrant.configure("2") do |config| |
||||
|
||||
config.vm.provider "virtualbox" do |v| |
||||
v.memory = 4096 |
||||
v.cpus = 2 |
||||
end |
||||
|
||||
config.vm.define "ubuntu14" do |box| |
||||
box.vm.box = "ubuntu/trusty64" |
||||
end |
||||
|
||||
config.vm.define "centos65" do |box| |
||||
box.vm.box = "chef/centos-6.5" |
||||
end |
||||
|
||||
end |
||||
@ -1,74 +0,0 @@ |
||||
#!/bin/sh |
||||
# |
||||
# Set environment variables so that we can compile leveldb using |
||||
# fbcode settings. It uses the latest g++ compiler and also |
||||
# uses jemalloc |
||||
|
||||
TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f |
||||
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native" |
||||
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1" |
||||
TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3 |
||||
GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1 |
||||
|
||||
# location of libgcc |
||||
LIBGCC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include" |
||||
LIBGCC_LIBS=" -L $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/libs" |
||||
|
||||
# location of glibc |
||||
GLIBC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include" |
||||
GLIBC_LIBS=" -L $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib" |
||||
|
||||
# location of snappy headers and libraries |
||||
SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include" |
||||
SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a" |
||||
|
||||
# location of zlib headers and libraries |
||||
ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include" |
||||
ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a" |
||||
|
||||
# location of gflags headers and libraries |
||||
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include" |
||||
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a" |
||||
|
||||
# location of bzip headers and libraries |
||||
BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/include" |
||||
BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/lib/libbz2.a" |
||||
|
||||
# location of gflags headers and libraries |
||||
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include" |
||||
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a" |
||||
|
||||
# use Intel SSE support for checksum calculations |
||||
export USE_SSE=" -msse -msse4.2 " |
||||
|
||||
CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.2/0b7c69d/bin/clang $CLANG_INCLUDES" |
||||
CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.2/0b7c69d/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $GFLAGS_INCLUDE" |
||||
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar |
||||
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib |
||||
|
||||
CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin -nostdlib " |
||||
CFLAGS+=" -nostdinc -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1 " |
||||
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/x86_64-facebook-linux " |
||||
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/backward " |
||||
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include " |
||||
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include " |
||||
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/clang/clang-3.2/0b7c69d/lib/clang/3.2/include " |
||||
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include/linux " |
||||
CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include " |
||||
CFLAGS+=" -Wall -Wno-sign-compare -Wno-unused-variable -Winvalid-pch -Wno-deprecated -Woverloaded-virtual" |
||||
CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE" |
||||
CXXFLAGS="$CFLAGS -nostdinc++" |
||||
|
||||
CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC" |
||||
|
||||
EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a" |
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a" |
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS" |
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2" |
||||
EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin" |
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS " |
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $GFLAGS_LIBS" |
||||
|
||||
export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED |
||||
@ -1,70 +0,0 @@ |
||||
#!/bin/sh |
||||
# |
||||
# Set environment variables so that we can compile leveldb using |
||||
# fbcode settings. It uses the latest g++ compiler and also |
||||
# uses jemalloc |
||||
|
||||
TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f |
||||
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native" |
||||
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1" |
||||
TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3 |
||||
|
||||
# location of libhdfs libraries |
||||
if test "$USE_HDFS"; then |
||||
JAVA_HOME="/usr/local/jdk-6u22-64" |
||||
JINCLUDE="-I$JAVA_HOME/include -I$JAVA_HOME/include/linux" |
||||
GLIBC_RUNTIME_PATH="/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1" |
||||
HDFSLIB=" -Wl,--no-whole-archive hdfs/libhdfs.a -L$JAVA_HOME/jre/lib/amd64 " |
||||
HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib " |
||||
HDFSLIB+=" -ldl -lverify -ljava -ljvm " |
||||
fi |
||||
|
||||
# location of libgcc |
||||
LIBGCC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include" |
||||
LIBGCC_LIBS=" -L $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/libs" |
||||
|
||||
# location of glibc |
||||
GLIBC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include" |
||||
GLIBC_LIBS=" -L $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib" |
||||
|
||||
# location of snappy headers and libraries |
||||
SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include" |
||||
SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a" |
||||
|
||||
# location of zlib headers and libraries |
||||
ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include" |
||||
ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a" |
||||
|
||||
# location of bzip headers and libraries |
||||
BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/include" |
||||
BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/lib/libbz2.a" |
||||
|
||||
# location of gflags headers and libraries |
||||
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include" |
||||
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a" |
||||
|
||||
# use Intel SSE support for checksum calculations |
||||
export USE_SSE=" -msse -msse4.2 " |
||||
|
||||
CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/gcc" |
||||
CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $GFLAGS_INCLUDE" |
||||
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar |
||||
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib |
||||
|
||||
CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic" |
||||
CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC" |
||||
CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE" |
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT" |
||||
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2" |
||||
|
||||
EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a" |
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a" |
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS" |
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS " |
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS" |
||||
|
||||
VALGRIND_VER="$TOOLCHAIN_LIB_BASE/valgrind/valgrind-3.8.1/91ddd43/bin/" |
||||
|
||||
export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER |
||||
@ -1,86 +0,0 @@ |
||||
#!/bin/sh |
||||
# |
||||
# Set environment variables so that we can compile rocksdb using |
||||
# fbcode settings. It uses the latest g++ compiler and also |
||||
# uses jemalloc |
||||
|
||||
TOOLCHAIN_REV=53dc1fe83f84e9145b9ffb81b81aa7f6a49c87cc |
||||
CENTOS_VERSION=`rpm -q --qf "%{VERSION}" $(rpm -q --whatprovides redhat-release)` |
||||
if [ "$CENTOS_VERSION" = "6" ]; then |
||||
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos6-native" |
||||
else |
||||
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native" |
||||
fi |
||||
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.8.1-glibc-2.17" |
||||
|
||||
# location of libhdfs libraries |
||||
if test "$USE_HDFS"; then |
||||
JAVA_HOME="/usr/local/jdk-6u22-64" |
||||
JINCLUDE="-I$JAVA_HOME/include -I$JAVA_HOME/include/linux" |
||||
GLIBC_RUNTIME_PATH="/usr/local/fbcode/gcc-4.8.1-glibc-2.17" |
||||
HDFSLIB=" -Wl,--no-whole-archive hdfs/libhdfs.a -L$JAVA_HOME/jre/lib/amd64 " |
||||
HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib " |
||||
HDFSLIB+=" -ldl -lverify -ljava -ljvm " |
||||
fi |
||||
|
||||
# location of libgcc |
||||
LIBGCC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.8.1/8aac7fc/include" |
||||
LIBGCC_LIBS=" -L $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.8.1/8aac7fc/libs" |
||||
|
||||
# location of glibc |
||||
GLIBC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/glibc/glibc-2.17/99df8fc/include" |
||||
GLIBC_LIBS=" -L $TOOLCHAIN_LIB_BASE/glibc/glibc-2.17/99df8fc/lib" |
||||
|
||||
# location of snappy headers and libraries |
||||
SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/43d84e2/include" |
||||
SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/43d84e2/lib/libsnappy.a" |
||||
|
||||
# location of zlib headers and libraries |
||||
ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/c3f970a/include" |
||||
ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/c3f970a/lib/libz.a" |
||||
|
||||
# location of bzip headers and libraries |
||||
BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/c3f970a/include" |
||||
BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/c3f970a/lib/libbz2.a" |
||||
|
||||
LZ4_REV=065ec7e38fe83329031f6668c43bef83eff5808b |
||||
LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/include" |
||||
LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a" |
||||
|
||||
# location of gflags headers and libraries |
||||
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/c3f970a/include" |
||||
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/c3f970a/lib/libgflags.a" |
||||
|
||||
# location of jemalloc |
||||
JEMALLOC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-3.4.1/4d53c6f/include/" |
||||
JEMALLOC_LIB=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-3.4.1/4d53c6f/lib/libjemalloc.a" |
||||
|
||||
# location of numa |
||||
NUMA_REV=829d10dac0230f99cd7e1778869d2adf3da24b65 |
||||
NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/include/" |
||||
NUMA_LIB=" /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libnuma.a" |
||||
|
||||
# use Intel SSE support for checksum calculations |
||||
export USE_SSE=" -msse -msse4.2 " |
||||
|
||||
CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.8.1/cc6c9dc/bin/gcc" |
||||
CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.8.1/cc6c9dc/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE" |
||||
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar |
||||
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib |
||||
|
||||
CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic" |
||||
CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE" |
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT" |
||||
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DNUMA" |
||||
|
||||
EXEC_LDFLAGS="-Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so" |
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/675d945/lib/libunwind.a" |
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB" |
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS " |
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS" |
||||
|
||||
VALGRIND_VER="$TOOLCHAIN_LIB_BASE/valgrind/valgrind-3.8.1/c3f970a/bin/" |
||||
|
||||
export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE |
||||
@ -0,0 +1,125 @@ |
||||
#!/bin/sh |
||||
# |
||||
# Set environment variables so that we can compile rocksdb using |
||||
# fbcode settings. It uses the latest g++ and clang compilers and also |
||||
# uses jemalloc |
||||
# Environment variables that change the behavior of this script: |
||||
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included |
||||
|
||||
CFLAGS="" |
||||
|
||||
# location of libgcc |
||||
LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/0473c80518a10d6efcbe24c5eeca3fb4ec9b519c/4.9.x/gcc-4.9-glibc-2.20/e1a7e4e" |
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include" |
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/libs" |
||||
|
||||
# location of glibc |
||||
GLIBC_REV=7397bed99280af5d9543439cdb7d018af7542720 |
||||
GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/include" |
||||
GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/lib" |
||||
|
||||
# location of snappy headers and libraries |
||||
SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/include/" |
||||
if test -z $PIC_BUILD; then |
||||
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy.a" |
||||
else |
||||
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy_pic.a" |
||||
fi |
||||
CFLAGS+=" -DSNAPPY" |
||||
|
||||
if test -z $PIC_BUILD; then |
||||
# location of zlib headers and libraries |
||||
ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/include/" |
||||
ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/lib/libz.a" |
||||
CFLAGS+=" -DZLIB" |
||||
|
||||
# location of bzip headers and libraries |
||||
BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/include/" |
||||
BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/lib/libbz2.a" |
||||
CFLAGS+=" -DBZIP2" |
||||
|
||||
LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/79d2943e2dd7208a3e0b06cf95e9f85f05fe9e1b/r124/gcc-4.9-glibc-2.20/4230243/include/" |
||||
LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/79d2943e2dd7208a3e0b06cf95e9f85f05fe9e1b/r124/gcc-4.9-glibc-2.20/4230243/lib/liblz4.a" |
||||
CFLAGS+=" -DLZ4" |
||||
fi |
||||
|
||||
# location of gflags headers and libraries |
||||
GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/include/" |
||||
if test -z $PIC_BUILD; then |
||||
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags.a" |
||||
else |
||||
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags_pic.a" |
||||
fi |
||||
CFLAGS+=" -DGFLAGS=google" |
||||
|
||||
# location of jemalloc |
||||
JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/include/" |
||||
JEMALLOC_LIB=" -Wl,--whole-archive /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/lib/libjemalloc.a" |
||||
|
||||
if test -z $PIC_BUILD; then |
||||
# location of numa |
||||
NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/include/" |
||||
NUMA_LIB=" /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/lib/libnuma.a" |
||||
CFLAGS+=" -DNUMA" |
||||
|
||||
# location of libunwind |
||||
LIBUNWIND="/mnt/gvfs/third-party2/libunwind/1de3b75e0afedfe5585b231bbb340ec7a1542335/1.1/gcc-4.9-glibc-2.20/34235e8/lib/libunwind.a" |
||||
fi |
||||
|
||||
# use Intel SSE support for checksum calculations |
||||
export USE_SSE=1 |
||||
|
||||
BINUTILS="/mnt/gvfs/third-party2/binutils/0b6ad0c88ddd903333a48ae8bff134efac468e4a/2.25/centos6-native/da39a3e/bin" |
||||
AR="$BINUTILS/ar" |
||||
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE" |
||||
|
||||
GCC_BASE="/mnt/gvfs/third-party2/gcc/1c67a0b88f64d4d9ced0382d141c76aaa7d62fba/4.9.x/centos6-native/1317bc4" |
||||
STDLIBS="-L $GCC_BASE/lib64" |
||||
|
||||
CLANG_BASE="/mnt/gvfs/third-party2/clang/290704c112bf894bf4a30d7bbd1be81e34998473/dev" |
||||
CLANG_ANALYZER="$CLANG_BASE/centos6-native/af4b1a0/bin/clang++" |
||||
CLANG_SCAN_BUILD="$CLANG_BASE/src/clang/tools/scan-build/scan-build" |
||||
|
||||
if [ -z "$USE_CLANG" ]; then |
||||
# gcc |
||||
CC="$GCC_BASE/bin/gcc" |
||||
CXX="$GCC_BASE/bin/g++" |
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold" |
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE" |
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE" |
||||
else |
||||
# clang |
||||
CLANG_INCLUDE="$CLANG_BASE/gcc-4.9-glibc-2.20/74c386f/lib/clang/dev/include/" |
||||
CC="$CLANG_BASE/centos6-native/af4b1a0/bin/clang" |
||||
CXX="$CLANG_BASE/centos6-native/af4b1a0/bin/clang++" |
||||
|
||||
KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/ffd14f660a43c4b92717986b1bba66722ef089d0/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.9-glibc-2.20/da39a3e/include" |
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib" |
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x " |
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x/x86_64-facebook-linux " |
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE" |
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE" |
||||
CFLAGS+=" -isystem $CLANG_INCLUDE" |
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " |
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " |
||||
CXXFLAGS="-nostdinc++" |
||||
fi |
||||
|
||||
CFLAGS+=" $DEPS_INCLUDE" |
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_FALLOCATE_PRESENT" |
||||
CXXFLAGS+=" $CFLAGS" |
||||
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB" |
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so" |
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $LIBUNWIND" |
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" |
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS" |
||||
|
||||
VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/6c45ef049cbf11c2df593addb712cd891049e737/3.10.0/gcc-4.9-glibc-2.20/4230243/bin/" |
||||
|
||||
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD |
||||
@ -0,0 +1,105 @@ |
||||
#!/bin/sh |
||||
# |
||||
# Set environment variables so that we can compile rocksdb using |
||||
# fbcode settings. It uses the latest g++ compiler and also |
||||
# uses jemalloc |
||||
|
||||
# location of libgcc |
||||
LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/7712e757d7355cb51292454ee0b7b46a467fdfed/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc" |
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include" |
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/libs" |
||||
|
||||
# location of glibc |
||||
GLIBC_REV=6e40560b4e0b6d690fd1cf8c7a43ad7452b04cfa |
||||
GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/include" |
||||
GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/lib" |
||||
|
||||
# location of snappy headers and libraries |
||||
SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/aef17f6c0b44b4fe408bd06f67c93701ab0a6ceb/1.0.3/gcc-4.8.1-glibc-2.17/43d84e2/include" |
||||
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/aef17f6c0b44b4fe408bd06f67c93701ab0a6ceb/1.0.3/gcc-4.8.1-glibc-2.17/43d84e2/lib/libsnappy.a" |
||||
|
||||
# location of zlib headers and libraries |
||||
ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/25c6216928b4d77b59ddeca0990ff6fe9ac16b81/1.2.5/gcc-4.8.1-glibc-2.17/c3f970a/include" |
||||
ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/25c6216928b4d77b59ddeca0990ff6fe9ac16b81/1.2.5/gcc-4.8.1-glibc-2.17/c3f970a/lib/libz.a" |
||||
|
||||
# location of bzip headers and libraries |
||||
BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/c9ef7629c2aa0024f7a416e87602f06eb88f5eac/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/include/" |
||||
BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/c9ef7629c2aa0024f7a416e87602f06eb88f5eac/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/lib/libbz2.a" |
||||
|
||||
LZ4_REV=065ec7e38fe83329031f6668c43bef83eff5808b |
||||
LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/include" |
||||
LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a" |
||||
|
||||
# location of gflags headers and libraries |
||||
GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/1ad047a6e6f6673991918ecadc670868205a243a/1.6/gcc-4.8.1-glibc-2.17/c3f970a/include/" |
||||
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/1ad047a6e6f6673991918ecadc670868205a243a/1.6/gcc-4.8.1-glibc-2.17/c3f970a/lib/libgflags.a" |
||||
|
||||
# location of jemalloc |
||||
JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/c60d854f7824f334195fe7fd34b2bc9057e3c1f9/3.6.0/gcc-4.8.1-glibc-2.17/4d53c6f/include" |
||||
JEMALLOC_LIB=" -Wl,--whole-archive /mnt/gvfs/third-party2/jemalloc/c60d854f7824f334195fe7fd34b2bc9057e3c1f9/3.6.0/gcc-4.8.1-glibc-2.17/4d53c6f/lib/libjemalloc.a" |
||||
|
||||
# location of numa |
||||
NUMA_REV=829d10dac0230f99cd7e1778869d2adf3da24b65 |
||||
NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/include/" |
||||
NUMA_LIB=" /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libnuma.a" |
||||
|
||||
# location of libunwind |
||||
LIBUNWIND_REV=2c060e64064559905d46fd194000d61592087bdc |
||||
LIBUNWIND="/mnt/gvfs/third-party2/libunwind/$LIBUNWIND_REV/1.1/gcc-4.8.1-glibc-2.17/675d945/lib/libunwind.a" |
||||
|
||||
# use Intel SSE support for checksum calculations |
||||
export USE_SSE=1 |
||||
|
||||
BINUTILS="/mnt/gvfs/third-party2/binutils/2aff2e7b474cd3e6ab23495ad1224b7d214b9f8e/2.21.1/centos6-native/da39a3e/bin" |
||||
AR="$BINUTILS/ar" |
||||
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE" |
||||
|
||||
GCC_BASE="/mnt/gvfs/third-party2/gcc/1ec615e23800f0815d474478ba476a0adc3fe788/4.8.1/centos6-native/cc6c9dc" |
||||
STDLIBS="-L $GCC_BASE/lib64" |
||||
|
||||
if [ -z "$USE_CLANG" ]; then |
||||
# gcc |
||||
CC="$GCC_BASE/bin/gcc" |
||||
CXX="$GCC_BASE/bin/g++" |
||||
|
||||
CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic" |
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE" |
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE" |
||||
else |
||||
# clang |
||||
CLANG_BASE="/mnt/gvfs/third-party2/clang/9ab68376f938992c4eb5946ca68f90c3185cffc8/3.4" |
||||
CLANG_INCLUDE="$CLANG_BASE/gcc-4.8.1-glibc-2.17/fb0f730/lib/clang/3.4/include" |
||||
CC="$CLANG_BASE/centos6-native/9cefd8a/bin/clang" |
||||
CXX="$CLANG_BASE/centos6-native/9cefd8a/bin/clang++" |
||||
|
||||
KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/a683ed7135276731065a9d76d3016c9731f4e2f9/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.8.1-glibc-2.17/da39a3e/include/" |
||||
|
||||
CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib" |
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 " |
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux " |
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE" |
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE" |
||||
CFLAGS+=" -isystem $CLANG_INCLUDE" |
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux " |
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE " |
||||
CXXFLAGS="-nostdinc++" |
||||
fi |
||||
|
||||
CFLAGS+=" $DEPS_INCLUDE" |
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_FALLOCATE_PRESENT" |
||||
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DNUMA" |
||||
CXXFLAGS+=" $CFLAGS" |
||||
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB" |
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so" |
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $LIBUNWIND" |
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++" |
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS" |
||||
|
||||
VALGRIND_REV=b2a9f85e4b70cd03abc85a7f3027fbc4cef35bd0 |
||||
VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/$VALGRIND_REV/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a/bin/" |
||||
|
||||
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE |
||||
@ -1,25 +0,0 @@ |
||||
#!/bin/sh |
||||
# Install gflags for mac developers. |
||||
|
||||
set -e |
||||
|
||||
DIR=`mktemp -d /tmp/rocksdb_gflags_XXXX` |
||||
|
||||
cd $DIR |
||||
wget https://gflags.googlecode.com/files/gflags-2.0.tar.gz |
||||
tar xvfz gflags-2.0.tar.gz |
||||
cd gflags-2.0 |
||||
|
||||
./configure |
||||
make |
||||
make install |
||||
|
||||
# Add include/lib path for g++ |
||||
echo 'export LIBRARY_PATH+=":/usr/local/lib"' >> ~/.bash_profile |
||||
echo 'export CPATH+=":/usr/local/include"' >> ~/.bash_profile |
||||
|
||||
echo "" |
||||
echo "-----------------------------------------------------------------------------" |
||||
echo "| Installation Completed |" |
||||
echo "-----------------------------------------------------------------------------" |
||||
echo "Please run \`. ~/.bash_profile\` to be able to compile with gflags" |
||||
@ -0,0 +1,116 @@ |
||||
#/usr/bin/env bash |
||||
|
||||
set -e |
||||
|
||||
function log() { |
||||
echo "[+] $1" |
||||
} |
||||
|
||||
function fatal() { |
||||
echo "[!] $1" |
||||
exit 1 |
||||
} |
||||
|
||||
function platform() { |
||||
local __resultvar=$1 |
||||
if [[ -f "/etc/yum.conf" ]]; then |
||||
eval $__resultvar="centos" |
||||
elif [[ -f "/etc/dpkg/dpkg.cfg" ]]; then |
||||
eval $__resultvar="ubuntu" |
||||
else |
||||
fatal "Unknwon operating system" |
||||
fi |
||||
} |
||||
platform OS |
||||
|
||||
function package() { |
||||
if [[ $OS = "ubuntu" ]]; then |
||||
if dpkg --get-selections | grep --quiet $1; then |
||||
log "$1 is already installed. skipping." |
||||
else |
||||
apt-get install $@ -y |
||||
fi |
||||
elif [[ $OS = "centos" ]]; then |
||||
if rpm -qa | grep --quiet $1; then |
||||
log "$1 is already installed. skipping." |
||||
else |
||||
yum install $@ -y |
||||
fi |
||||
fi |
||||
} |
||||
|
||||
function detect_fpm_output() { |
||||
if [[ $OS = "ubuntu" ]]; then |
||||
export FPM_OUTPUT=deb |
||||
elif [[ $OS = "centos" ]]; then |
||||
export FPM_OUTPUT=rpm |
||||
fi |
||||
} |
||||
detect_fpm_output |
||||
|
||||
function gem_install() { |
||||
if gem list | grep --quiet $1; then |
||||
log "$1 is already installed. skipping." |
||||
else |
||||
gem install $@ |
||||
fi |
||||
} |
||||
|
||||
function main() { |
||||
if [[ $# -ne 1 ]]; then |
||||
fatal "Usage: $0 <rocksdb_version>" |
||||
else |
||||
log "using rocksdb version: $1" |
||||
fi |
||||
|
||||
if [[ -d /vagrant ]]; then |
||||
if [[ $OS = "ubuntu" ]]; then |
||||
package g++-4.7 |
||||
export CXX=g++-4.7 |
||||
|
||||
# the deb would depend on libgflags2, but the static lib is the only thing |
||||
# installed by make install |
||||
package libgflags-dev |
||||
|
||||
package ruby-all-dev |
||||
elif [[ $OS = "centos" ]]; then |
||||
pushd /etc/yum.repos.d |
||||
if [[ ! -f /etc/yum.repos.d/devtools-1.1.repo ]]; then |
||||
wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo |
||||
fi |
||||
package devtoolset-1.1-gcc --enablerepo=testing-1.1-devtools-6 |
||||
package devtoolset-1.1-gcc-c++ --enablerepo=testing-1.1-devtools-6 |
||||
export CC=/opt/centos/devtoolset-1.1/root/usr/bin/gcc |
||||
export CPP=/opt/centos/devtoolset-1.1/root/usr/bin/cpp |
||||
export CXX=/opt/centos/devtoolset-1.1/root/usr/bin/c++ |
||||
export PATH=$PATH:/opt/centos/devtoolset-1.1/root/usr/bin |
||||
popd |
||||
if ! rpm -qa | grep --quiet gflags; then |
||||
rpm -i https://github.com/schuhschuh/gflags/releases/download/v2.1.0/gflags-devel-2.1.0-1.amd64.rpm |
||||
fi |
||||
|
||||
package ruby |
||||
package ruby-devel |
||||
package rubygems |
||||
package rpm-build |
||||
fi |
||||
fi |
||||
gem_install fpm |
||||
|
||||
make static_lib |
||||
make install INSTALL_PATH=package |
||||
fpm \ |
||||
-s dir \ |
||||
-t $FPM_OUTPUT \ |
||||
-n rocksdb \ |
||||
-v $1 \ |
||||
--prefix /usr \ |
||||
--url http://rocksdb.org/ \ |
||||
-m rocksdb@fb.com \ |
||||
--license BSD \ |
||||
--vendor Facebook \ |
||||
--description "RocksDB is an embeddable persistent key-value store for fast storage." \ |
||||
package |
||||
} |
||||
|
||||
main $@ |
||||
@ -0,0 +1,14 @@ |
||||
#!/bin/sh |
||||
if [ "$#" = "0" ]; then |
||||
echo "Usage: $0 major|minor|patch" |
||||
exit 1 |
||||
fi |
||||
if [ "$1" = "major" ]; then |
||||
cat include/rocksdb/version.h | grep MAJOR | head -n1 | awk '{print $3}' |
||||
fi |
||||
if [ "$1" = "minor" ]; then |
||||
cat include/rocksdb/version.h | grep MINOR | head -n1 | awk '{print $3}' |
||||
fi |
||||
if [ "$1" = "patch" ]; then |
||||
cat include/rocksdb/version.h | grep PATCH | head -n1 | awk '{print $3}' |
||||
fi |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,127 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
#pragma once |
||||
|
||||
#include <atomic> |
||||
#include <deque> |
||||
#include <limits> |
||||
#include <set> |
||||
#include <utility> |
||||
#include <vector> |
||||
#include <string> |
||||
#include <functional> |
||||
|
||||
#include "db/dbformat.h" |
||||
#include "db/log_writer.h" |
||||
#include "db/snapshot.h" |
||||
#include "db/column_family.h" |
||||
#include "db/version_edit.h" |
||||
#include "db/memtable_list.h" |
||||
#include "port/port.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/memtablerep.h" |
||||
#include "rocksdb/compaction_filter.h" |
||||
#include "rocksdb/transaction_log.h" |
||||
#include "util/autovector.h" |
||||
#include "util/stop_watch.h" |
||||
#include "util/thread_local.h" |
||||
#include "util/scoped_arena_iterator.h" |
||||
#include "db/internal_stats.h" |
||||
#include "db/write_controller.h" |
||||
#include "db/flush_scheduler.h" |
||||
#include "db/write_thread.h" |
||||
#include "db/job_context.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class MemTable; |
||||
class TableCache; |
||||
class Version; |
||||
class VersionEdit; |
||||
class VersionSet; |
||||
class Arena; |
||||
|
||||
class CompactionJob { |
||||
public: |
||||
// TODO(icanadi) make effort to reduce number of parameters here
|
||||
// IMPORTANT: mutable_cf_options needs to be alive while CompactionJob is
|
||||
// alive
|
||||
CompactionJob(Compaction* compaction, const DBOptions& db_options, |
||||
const MutableCFOptions& mutable_cf_options, |
||||
const EnvOptions& env_options, VersionSet* versions, |
||||
std::atomic<bool>* shutting_down, LogBuffer* log_buffer, |
||||
Directory* db_directory, Directory* output_directory, |
||||
Statistics* stats, SnapshotList* snapshot_list, |
||||
bool is_snapshot_supported, std::shared_ptr<Cache> table_cache, |
||||
std::function<uint64_t()> yield_callback); |
||||
|
||||
~CompactionJob() { assert(compact_ == nullptr); } |
||||
|
||||
// no copy/move
|
||||
CompactionJob(CompactionJob&& job) = delete; |
||||
CompactionJob(const CompactionJob& job) = delete; |
||||
CompactionJob& operator=(const CompactionJob& job) = delete; |
||||
|
||||
// REQUIRED: mutex held
|
||||
void Prepare(); |
||||
// REQUIRED mutex not held
|
||||
Status Run(); |
||||
// REQUIRED: mutex held
|
||||
// status is the return of Run()
|
||||
void Install(Status* status, InstrumentedMutex* db_mutex); |
||||
|
||||
private: |
||||
void AllocateCompactionOutputFileNumbers(); |
||||
// Call compaction filter if is_compaction_v2 is not true. Then iterate
|
||||
// through input and compact the kv-pairs
|
||||
Status ProcessKeyValueCompaction(int64_t* imm_micros, Iterator* input, |
||||
bool is_compaction_v2); |
||||
// Call compaction_filter_v2->Filter() on kv-pairs in compact
|
||||
void CallCompactionFilterV2(CompactionFilterV2* compaction_filter_v2); |
||||
Status FinishCompactionOutputFile(Iterator* input); |
||||
Status InstallCompactionResults(InstrumentedMutex* db_mutex); |
||||
SequenceNumber findEarliestVisibleSnapshot( |
||||
SequenceNumber in, const std::vector<SequenceNumber>& snapshots, |
||||
SequenceNumber* prev_snapshot); |
||||
void RecordCompactionIOStats(); |
||||
Status OpenCompactionOutputFile(); |
||||
void CleanupCompaction(const Status& status); |
||||
|
||||
// CompactionJob state
|
||||
struct CompactionState; |
||||
CompactionState* compact_; |
||||
|
||||
bool bottommost_level_; |
||||
SequenceNumber earliest_snapshot_; |
||||
SequenceNumber visible_at_tip_; |
||||
SequenceNumber latest_snapshot_; |
||||
|
||||
InternalStats::CompactionStats compaction_stats_; |
||||
|
||||
// DBImpl state
|
||||
const DBOptions& db_options_; |
||||
const MutableCFOptions& mutable_cf_options_; |
||||
const EnvOptions& env_options_; |
||||
Env* env_; |
||||
VersionSet* versions_; |
||||
std::atomic<bool>* shutting_down_; |
||||
LogBuffer* log_buffer_; |
||||
Directory* db_directory_; |
||||
Directory* output_directory_; |
||||
Statistics* stats_; |
||||
SnapshotList* snapshots_; |
||||
bool is_snapshot_supported_; |
||||
std::shared_ptr<Cache> table_cache_; |
||||
|
||||
// yield callback
|
||||
std::function<uint64_t()> yield_callback_; |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,183 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <map> |
||||
#include <string> |
||||
|
||||
#include "db/compaction_job.h" |
||||
#include "db/column_family.h" |
||||
#include "db/version_set.h" |
||||
#include "db/writebuffer.h" |
||||
#include "rocksdb/cache.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/db.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
#include "table/mock_table.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
// TODO(icanadi) Make it simpler once we mock out VersionSet
|
||||
class CompactionJobTest { |
||||
public: |
||||
CompactionJobTest() |
||||
: env_(Env::Default()), |
||||
dbname_(test::TmpDir() + "/compaction_job_test"), |
||||
mutable_cf_options_(Options(), ImmutableCFOptions(Options())), |
||||
table_cache_(NewLRUCache(50000, 16, 8)), |
||||
write_buffer_(db_options_.db_write_buffer_size), |
||||
versions_(new VersionSet(dbname_, &db_options_, env_options_, |
||||
table_cache_.get(), &write_buffer_, |
||||
&write_controller_)), |
||||
shutting_down_(false), |
||||
mock_table_factory_(new mock::MockTableFactory()) { |
||||
ASSERT_OK(env_->CreateDirIfMissing(dbname_)); |
||||
db_options_.db_paths.emplace_back(dbname_, |
||||
std::numeric_limits<uint64_t>::max()); |
||||
NewDB(); |
||||
std::vector<ColumnFamilyDescriptor> column_families; |
||||
cf_options_.table_factory = mock_table_factory_; |
||||
column_families.emplace_back(kDefaultColumnFamilyName, cf_options_); |
||||
|
||||
|
||||
ASSERT_OK(versions_->Recover(column_families, false)); |
||||
} |
||||
|
||||
std::string GenerateFileName(uint64_t file_number) { |
||||
FileMetaData meta; |
||||
std::vector<DbPath> db_paths; |
||||
db_paths.emplace_back(dbname_, std::numeric_limits<uint64_t>::max()); |
||||
meta.fd = FileDescriptor(file_number, 0, 0); |
||||
return TableFileName(db_paths, meta.fd.GetNumber(), meta.fd.GetPathId()); |
||||
} |
||||
|
||||
// returns expected result after compaction
|
||||
mock::MockFileContents CreateTwoFiles() { |
||||
mock::MockFileContents expected_results; |
||||
const int kKeysPerFile = 10000; |
||||
SequenceNumber sequence_number = 0; |
||||
for (int i = 0; i < 2; ++i) { |
||||
mock::MockFileContents contents; |
||||
SequenceNumber smallest_seqno = 0, largest_seqno = 0; |
||||
InternalKey smallest, largest; |
||||
for (int k = 0; k < kKeysPerFile; ++k) { |
||||
auto key = ToString(i * (kKeysPerFile / 2) + k); |
||||
auto value = ToString(i * kKeysPerFile + k); |
||||
InternalKey internal_key(key, ++sequence_number, kTypeValue); |
||||
if (k == 0) { |
||||
smallest = internal_key; |
||||
smallest_seqno = sequence_number; |
||||
} else if (k == kKeysPerFile - 1) { |
||||
largest = internal_key; |
||||
largest_seqno = sequence_number; |
||||
} |
||||
std::pair<std::string, std::string> key_value( |
||||
{internal_key.Encode().ToString(), value}); |
||||
contents.insert(key_value); |
||||
if (i == 1 || k < kKeysPerFile / 2) { |
||||
expected_results.insert(key_value); |
||||
} |
||||
} |
||||
|
||||
uint64_t file_number = versions_->NewFileNumber(); |
||||
ASSERT_OK(mock_table_factory_->CreateMockTable( |
||||
env_, GenerateFileName(file_number), std::move(contents))); |
||||
|
||||
VersionEdit edit; |
||||
edit.AddFile(0, file_number, 0, 10, smallest, largest, smallest_seqno, |
||||
largest_seqno); |
||||
|
||||
mutex_.Lock(); |
||||
versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(), |
||||
mutable_cf_options_, &edit, &mutex_); |
||||
mutex_.Unlock(); |
||||
} |
||||
versions_->SetLastSequence(sequence_number); |
||||
return expected_results; |
||||
} |
||||
|
||||
void NewDB() { |
||||
VersionEdit new_db; |
||||
new_db.SetLogNumber(0); |
||||
new_db.SetNextFile(2); |
||||
new_db.SetLastSequence(0); |
||||
|
||||
const std::string manifest = DescriptorFileName(dbname_, 1); |
||||
unique_ptr<WritableFile> file; |
||||
Status s = env_->NewWritableFile( |
||||
manifest, &file, env_->OptimizeForManifestWrite(env_options_)); |
||||
ASSERT_OK(s); |
||||
{ |
||||
log::Writer log(std::move(file)); |
||||
std::string record; |
||||
new_db.EncodeTo(&record); |
||||
s = log.AddRecord(record); |
||||
} |
||||
ASSERT_OK(s); |
||||
// Make "CURRENT" file that points to the new manifest file.
|
||||
s = SetCurrentFile(env_, dbname_, 1, nullptr); |
||||
} |
||||
|
||||
Env* env_; |
||||
std::string dbname_; |
||||
EnvOptions env_options_; |
||||
MutableCFOptions mutable_cf_options_; |
||||
std::shared_ptr<Cache> table_cache_; |
||||
WriteController write_controller_; |
||||
DBOptions db_options_; |
||||
ColumnFamilyOptions cf_options_; |
||||
WriteBuffer write_buffer_; |
||||
std::unique_ptr<VersionSet> versions_; |
||||
InstrumentedMutex mutex_; |
||||
std::atomic<bool> shutting_down_; |
||||
std::shared_ptr<mock::MockTableFactory> mock_table_factory_; |
||||
}; |
||||
|
||||
TEST(CompactionJobTest, Simple) { |
||||
auto cfd = versions_->GetColumnFamilySet()->GetDefault(); |
||||
|
||||
auto expected_results = CreateTwoFiles(); |
||||
|
||||
auto files = cfd->current()->storage_info()->LevelFiles(0); |
||||
ASSERT_EQ(2U, files.size()); |
||||
|
||||
std::unique_ptr<Compaction> compaction(Compaction::TEST_NewCompaction( |
||||
7, 0, 1, 1024 * 1024, 10, 0, kNoCompression)); |
||||
compaction->SetInputVersion(cfd->current()); |
||||
|
||||
auto compaction_input_files = compaction->TEST_GetInputFiles(0); |
||||
compaction_input_files->level = 0; |
||||
compaction_input_files->files.push_back(files[0]); |
||||
compaction_input_files->files.push_back(files[1]); |
||||
|
||||
SnapshotList snapshots; |
||||
int yield_callback_called = 0; |
||||
std::function<uint64_t()> yield_callback = [&]() { |
||||
yield_callback_called++; |
||||
return 0; |
||||
}; |
||||
LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get()); |
||||
mutex_.Lock(); |
||||
CompactionJob compaction_job(compaction.get(), db_options_, |
||||
*cfd->GetLatestMutableCFOptions(), env_options_, |
||||
versions_.get(), &shutting_down_, &log_buffer, |
||||
nullptr, nullptr, nullptr, &snapshots, true, |
||||
table_cache_, std::move(yield_callback)); |
||||
compaction_job.Prepare(); |
||||
mutex_.Unlock(); |
||||
ASSERT_OK(compaction_job.Run()); |
||||
mutex_.Lock(); |
||||
Status s; |
||||
compaction_job.Install(&s, &mutex_); |
||||
ASSERT_OK(s); |
||||
mutex_.Unlock(); |
||||
|
||||
mock_table_factory_->AssertLatestFile(expected_results); |
||||
ASSERT_EQ(yield_callback_called, 20000); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,259 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include "db/compaction_picker.h" |
||||
#include <limits> |
||||
#include <string> |
||||
#include "util/logging.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class CountingLogger : public Logger { |
||||
public: |
||||
using Logger::Logv; |
||||
virtual void Logv(const char* format, va_list ap) override { log_count++; } |
||||
size_t log_count; |
||||
}; |
||||
|
||||
class CompactionPickerTest { |
||||
public: |
||||
const Comparator* ucmp_; |
||||
InternalKeyComparator icmp_; |
||||
Options options_; |
||||
ImmutableCFOptions ioptions_; |
||||
MutableCFOptions mutable_cf_options_; |
||||
LevelCompactionPicker level_compaction_picker; |
||||
std::string cf_name_; |
||||
CountingLogger logger_; |
||||
LogBuffer log_buffer_; |
||||
uint32_t file_num_; |
||||
CompactionOptionsFIFO fifo_options_; |
||||
std::unique_ptr<VersionStorageInfo> vstorage_; |
||||
std::vector<std::unique_ptr<FileMetaData>> files_; |
||||
|
||||
CompactionPickerTest() |
||||
: ucmp_(BytewiseComparator()), |
||||
icmp_(ucmp_), |
||||
ioptions_(options_), |
||||
mutable_cf_options_(options_, ioptions_), |
||||
level_compaction_picker(ioptions_, &icmp_), |
||||
cf_name_("dummy"), |
||||
log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_), |
||||
file_num_(1), |
||||
vstorage_(nullptr) { |
||||
fifo_options_.max_table_files_size = 1; |
||||
mutable_cf_options_.RefreshDerivedOptions(ioptions_); |
||||
ioptions_.db_paths.emplace_back("dummy", |
||||
std::numeric_limits<uint64_t>::max()); |
||||
} |
||||
|
||||
~CompactionPickerTest() { |
||||
} |
||||
|
||||
void NewVersionStorage(int num_levels, CompactionStyle style) { |
||||
DeleteVersionStorage(); |
||||
options_.num_levels = num_levels; |
||||
vstorage_.reset(new VersionStorageInfo( |
||||
&icmp_, ucmp_, options_.num_levels, style, nullptr)); |
||||
} |
||||
|
||||
void DeleteVersionStorage() { |
||||
vstorage_.reset(); |
||||
files_.clear(); |
||||
} |
||||
|
||||
void Add(int level, uint32_t file_number, const char* smallest, |
||||
const char* largest, uint64_t file_size = 0, uint32_t path_id = 0, |
||||
SequenceNumber smallest_seq = 100, |
||||
SequenceNumber largest_seq = 100) { |
||||
assert(level < vstorage_->num_levels()); |
||||
FileMetaData* f = new FileMetaData; |
||||
f->fd = FileDescriptor(file_number, path_id, file_size); |
||||
f->smallest = InternalKey(smallest, smallest_seq, kTypeValue); |
||||
f->largest = InternalKey(largest, largest_seq, kTypeValue); |
||||
f->compensated_file_size = file_size; |
||||
f->refs = 0; |
||||
vstorage_->AddFile(level, f); |
||||
files_.emplace_back(f); |
||||
} |
||||
|
||||
void UpdateVersionStorageInfo() { |
||||
vstorage_->UpdateFilesBySize(); |
||||
vstorage_->UpdateNumNonEmptyLevels(); |
||||
vstorage_->GenerateFileIndexer(); |
||||
vstorage_->GenerateLevelFilesBrief(); |
||||
vstorage_->ComputeCompactionScore(mutable_cf_options_, fifo_options_); |
||||
vstorage_->SetFinalized(); |
||||
} |
||||
}; |
||||
|
||||
TEST(CompactionPickerTest, Empty) { |
||||
NewVersionStorage(6, kCompactionStyleLevel); |
||||
UpdateVersionStorageInfo(); |
||||
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( |
||||
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); |
||||
ASSERT_TRUE(compaction.get() == nullptr); |
||||
} |
||||
|
||||
TEST(CompactionPickerTest, Single) { |
||||
NewVersionStorage(6, kCompactionStyleLevel); |
||||
mutable_cf_options_.level0_file_num_compaction_trigger = 2; |
||||
Add(0, 1U, "p", "q"); |
||||
UpdateVersionStorageInfo(); |
||||
|
||||
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( |
||||
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); |
||||
ASSERT_TRUE(compaction.get() == nullptr); |
||||
} |
||||
|
||||
TEST(CompactionPickerTest, Level0Trigger) { |
||||
NewVersionStorage(6, kCompactionStyleLevel); |
||||
mutable_cf_options_.level0_file_num_compaction_trigger = 2; |
||||
Add(0, 1U, "150", "200"); |
||||
Add(0, 2U, "200", "250"); |
||||
|
||||
UpdateVersionStorageInfo(); |
||||
|
||||
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( |
||||
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); |
||||
ASSERT_TRUE(compaction.get() != nullptr); |
||||
ASSERT_EQ(2U, compaction->num_input_files(0)); |
||||
ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber()); |
||||
ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber()); |
||||
} |
||||
|
||||
TEST(CompactionPickerTest, Level1Trigger) { |
||||
NewVersionStorage(6, kCompactionStyleLevel); |
||||
Add(1, 66U, "150", "200", 1000000000U); |
||||
UpdateVersionStorageInfo(); |
||||
|
||||
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( |
||||
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); |
||||
ASSERT_TRUE(compaction.get() != nullptr); |
||||
ASSERT_EQ(1U, compaction->num_input_files(0)); |
||||
ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber()); |
||||
} |
||||
|
||||
TEST(CompactionPickerTest, Level1Trigger2) { |
||||
NewVersionStorage(6, kCompactionStyleLevel); |
||||
Add(1, 66U, "150", "200", 1000000001U); |
||||
Add(1, 88U, "201", "300", 1000000000U); |
||||
Add(2, 6U, "150", "179", 1000000000U); |
||||
Add(2, 7U, "180", "220", 1000000000U); |
||||
Add(2, 8U, "221", "300", 1000000000U); |
||||
UpdateVersionStorageInfo(); |
||||
|
||||
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( |
||||
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); |
||||
ASSERT_TRUE(compaction.get() != nullptr); |
||||
ASSERT_EQ(1U, compaction->num_input_files(0)); |
||||
ASSERT_EQ(2U, compaction->num_input_files(1)); |
||||
ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber()); |
||||
ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber()); |
||||
ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber()); |
||||
} |
||||
|
||||
TEST(CompactionPickerTest, LevelMaxScore) { |
||||
NewVersionStorage(6, kCompactionStyleLevel); |
||||
mutable_cf_options_.target_file_size_base = 10000000; |
||||
mutable_cf_options_.target_file_size_multiplier = 10; |
||||
Add(0, 1U, "150", "200", 1000000000U); |
||||
// Level 1 score 1.2
|
||||
Add(1, 66U, "150", "200", 6000000U); |
||||
Add(1, 88U, "201", "300", 6000000U); |
||||
// Level 2 score 1.8. File 7 is the largest. Should be picked
|
||||
Add(2, 6U, "150", "179", 60000000U); |
||||
Add(2, 7U, "180", "220", 60000001U); |
||||
Add(2, 8U, "221", "300", 60000000U); |
||||
// Level 3 score slightly larger than 1
|
||||
Add(3, 26U, "150", "170", 260000000U); |
||||
Add(3, 27U, "171", "179", 260000000U); |
||||
Add(3, 28U, "191", "220", 260000000U); |
||||
Add(3, 29U, "221", "300", 260000000U); |
||||
UpdateVersionStorageInfo(); |
||||
|
||||
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( |
||||
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_)); |
||||
ASSERT_TRUE(compaction.get() != nullptr); |
||||
ASSERT_EQ(1U, compaction->num_input_files(0)); |
||||
ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber()); |
||||
} |
||||
|
||||
TEST(CompactionPickerTest, NeedsCompactionLevel) { |
||||
const int kLevels = 6; |
||||
const int kFileCount = 20; |
||||
for (int level = 0; level < kLevels - 1; ++level) { |
||||
uint64_t file_size = |
||||
mutable_cf_options_.MaxBytesForLevel(level) * 2 / kFileCount; |
||||
for (int file_count = 1; file_count <= kFileCount; ++file_count) { |
||||
// start a brand new version in each test.
|
||||
NewVersionStorage(kLevels, kCompactionStyleLevel); |
||||
for (int i = 0; i < file_count; ++i) { |
||||
Add(level, i, ToString((i + 100) * 1000).c_str(), |
||||
ToString((i + 100) * 1000 + 999).c_str(), |
||||
file_size, 0, i * 100, i * 100 + 99); |
||||
} |
||||
UpdateVersionStorageInfo(); |
||||
ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level); |
||||
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()), |
||||
vstorage_->CompactionScore(0) >= 1); |
||||
// release the version storage
|
||||
DeleteVersionStorage(); |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST(CompactionPickerTest, NeedsCompactionUniversal) { |
||||
NewVersionStorage(1, kCompactionStyleUniversal); |
||||
UniversalCompactionPicker universal_compaction_picker( |
||||
ioptions_, &icmp_); |
||||
// must return false when there's no files.
|
||||
ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()), |
||||
false); |
||||
|
||||
// verify the trigger given different number of L0 files.
|
||||
for (int i = 1; |
||||
i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) { |
||||
Add(0, i, ToString((i + 100) * 1000).c_str(), |
||||
ToString((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100, |
||||
i * 100 + 99); |
||||
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()), |
||||
vstorage_->CompactionScore(0) >= 1); |
||||
} |
||||
} |
||||
|
||||
TEST(CompactionPickerTest, NeedsCompactionFIFO) { |
||||
NewVersionStorage(1, kCompactionStyleFIFO); |
||||
const int kFileCount = |
||||
mutable_cf_options_.level0_file_num_compaction_trigger * 3; |
||||
const uint64_t kFileSize = 100000; |
||||
const uint64_t kMaxSize = kFileSize * kFileCount / 2; |
||||
|
||||
fifo_options_.max_table_files_size = kMaxSize; |
||||
ioptions_.compaction_options_fifo = fifo_options_; |
||||
FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); |
||||
|
||||
// must return false when there's no files.
|
||||
ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), false); |
||||
|
||||
// verify whether compaction is needed based on the current
|
||||
// size of L0 files.
|
||||
uint64_t current_size = 0; |
||||
for (int i = 1; i <= kFileCount; ++i) { |
||||
Add(0, i, ToString((i + 100) * 1000).c_str(), |
||||
ToString((i + 100) * 1000 + 999).c_str(), |
||||
kFileSize, 0, i * 100, i * 100 + 99); |
||||
current_size += kFileSize; |
||||
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()), |
||||
vstorage_->CompactionScore(0) >= 1); |
||||
} |
||||
} |
||||
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } |
||||
@ -0,0 +1,434 @@ |
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
#include <map> |
||||
#include <string> |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "util/hash.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
#include "utilities/merge_operators.h" |
||||
|
||||
using std::unique_ptr; |
||||
|
||||
namespace rocksdb { |
||||
namespace { |
||||
|
||||
static const Comparator* comparator; |
||||
|
||||
// A comparator for std::map, using comparator
|
||||
struct MapComparator { |
||||
bool operator()(const std::string& a, const std::string& b) const { |
||||
return comparator->Compare(a, b) < 0; |
||||
} |
||||
}; |
||||
|
||||
typedef std::map<std::string, std::string, MapComparator> KVMap; |
||||
|
||||
class KVIter : public Iterator { |
||||
public: |
||||
explicit KVIter(const KVMap* map) : map_(map), iter_(map_->end()) {} |
||||
virtual bool Valid() const { return iter_ != map_->end(); } |
||||
virtual void SeekToFirst() { iter_ = map_->begin(); } |
||||
virtual void SeekToLast() { |
||||
if (map_->empty()) { |
||||
iter_ = map_->end(); |
||||
} else { |
||||
iter_ = map_->find(map_->rbegin()->first); |
||||
} |
||||
} |
||||
virtual void Seek(const Slice& k) { iter_ = map_->lower_bound(k.ToString()); } |
||||
virtual void Next() { ++iter_; } |
||||
virtual void Prev() { |
||||
if (iter_ == map_->begin()) { |
||||
iter_ = map_->end(); |
||||
return; |
||||
} |
||||
--iter_; |
||||
} |
||||
|
||||
virtual Slice key() const { return iter_->first; } |
||||
virtual Slice value() const { return iter_->second; } |
||||
virtual Status status() const { return Status::OK(); } |
||||
|
||||
private: |
||||
const KVMap* const map_; |
||||
KVMap::const_iterator iter_; |
||||
}; |
||||
|
||||
void AssertItersEqual(Iterator* iter1, Iterator* iter2) { |
||||
ASSERT_EQ(iter1->Valid(), iter2->Valid()); |
||||
if (iter1->Valid()) { |
||||
ASSERT_EQ(iter1->key().ToString(), iter2->key().ToString()); |
||||
ASSERT_EQ(iter1->value().ToString(), iter2->value().ToString()); |
||||
} |
||||
} |
||||
|
||||
// Measuring operations on DB (expect to be empty).
|
||||
// source_strings are candidate keys
|
||||
void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings, |
||||
Random* rnd, int num_writes, int num_iter_ops, |
||||
int num_trigger_flush) { |
||||
KVMap map; |
||||
|
||||
for (int i = 0; i < num_writes; i++) { |
||||
if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) { |
||||
db->Flush(FlushOptions()); |
||||
} |
||||
|
||||
int type = rnd->Uniform(2); |
||||
int index = rnd->Uniform(static_cast<int>(source_strings.size())); |
||||
auto& key = source_strings[index]; |
||||
switch (type) { |
||||
case 0: |
||||
// put
|
||||
map[key] = key; |
||||
ASSERT_OK(db->Put(WriteOptions(), key, key)); |
||||
break; |
||||
case 1: |
||||
// delete
|
||||
if (map.find(key) != map.end()) { |
||||
map.erase(key); |
||||
} |
||||
ASSERT_OK(db->Delete(WriteOptions(), key)); |
||||
break; |
||||
default: |
||||
assert(false); |
||||
} |
||||
} |
||||
|
||||
std::unique_ptr<Iterator> iter(db->NewIterator(ReadOptions())); |
||||
std::unique_ptr<Iterator> result_iter(new KVIter(&map)); |
||||
|
||||
bool is_valid = false; |
||||
for (int i = 0; i < num_iter_ops; i++) { |
||||
// Random walk and make sure iter and result_iter returns the
|
||||
// same key and value
|
||||
int type = rnd->Uniform(6); |
||||
ASSERT_OK(iter->status()); |
||||
switch (type) { |
||||
case 0: |
||||
// Seek to First
|
||||
iter->SeekToFirst(); |
||||
result_iter->SeekToFirst(); |
||||
break; |
||||
case 1: |
||||
// Seek to last
|
||||
iter->SeekToLast(); |
||||
result_iter->SeekToLast(); |
||||
break; |
||||
case 2: { |
||||
// Seek to random key
|
||||
auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size())); |
||||
auto key = source_strings[key_idx]; |
||||
iter->Seek(key); |
||||
result_iter->Seek(key); |
||||
break; |
||||
} |
||||
case 3: |
||||
// Next
|
||||
if (is_valid) { |
||||
iter->Next(); |
||||
result_iter->Next(); |
||||
} else { |
||||
continue; |
||||
} |
||||
break; |
||||
case 4: |
||||
// Prev
|
||||
if (is_valid) { |
||||
iter->Prev(); |
||||
result_iter->Prev(); |
||||
} else { |
||||
continue; |
||||
} |
||||
break; |
||||
default: { |
||||
assert(type == 5); |
||||
auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size())); |
||||
auto key = source_strings[key_idx]; |
||||
std::string result; |
||||
auto status = db->Get(ReadOptions(), key, &result); |
||||
if (map.find(key) == map.end()) { |
||||
ASSERT_TRUE(status.IsNotFound()); |
||||
} else { |
||||
ASSERT_EQ(map[key], result); |
||||
} |
||||
break; |
||||
} |
||||
} |
||||
AssertItersEqual(iter.get(), result_iter.get()); |
||||
is_valid = iter->Valid(); |
||||
} |
||||
} |
||||
|
||||
class DoubleComparator : public Comparator { |
||||
public: |
||||
DoubleComparator() {} |
||||
|
||||
virtual const char* Name() const { return "DoubleComparator"; } |
||||
|
||||
virtual int Compare(const Slice& a, const Slice& b) const { |
||||
double da = std::stod(a.ToString()); |
||||
double db = std::stod(b.ToString()); |
||||
if (da == db) { |
||||
return a.compare(b); |
||||
} else if (da > db) { |
||||
return 1; |
||||
} else { |
||||
return -1; |
||||
} |
||||
} |
||||
virtual void FindShortestSeparator(std::string* start, |
||||
const Slice& limit) const {} |
||||
|
||||
virtual void FindShortSuccessor(std::string* key) const {} |
||||
}; |
||||
|
||||
class HashComparator : public Comparator { |
||||
public: |
||||
HashComparator() {} |
||||
|
||||
virtual const char* Name() const { return "HashComparator"; } |
||||
|
||||
virtual int Compare(const Slice& a, const Slice& b) const { |
||||
uint32_t ha = Hash(a.data(), a.size(), 66); |
||||
uint32_t hb = Hash(b.data(), b.size(), 66); |
||||
if (ha == hb) { |
||||
return a.compare(b); |
||||
} else if (ha > hb) { |
||||
return 1; |
||||
} else { |
||||
return -1; |
||||
} |
||||
} |
||||
virtual void FindShortestSeparator(std::string* start, |
||||
const Slice& limit) const {} |
||||
|
||||
virtual void FindShortSuccessor(std::string* key) const {} |
||||
}; |
||||
|
||||
class TwoStrComparator : public Comparator { |
||||
public: |
||||
TwoStrComparator() {} |
||||
|
||||
virtual const char* Name() const { return "TwoStrComparator"; } |
||||
|
||||
virtual int Compare(const Slice& a, const Slice& b) const { |
||||
assert(a.size() >= 2); |
||||
assert(b.size() >= 2); |
||||
size_t size_a1 = static_cast<size_t>(a[0]); |
||||
size_t size_b1 = static_cast<size_t>(b[0]); |
||||
size_t size_a2 = static_cast<size_t>(a[1]); |
||||
size_t size_b2 = static_cast<size_t>(b[1]); |
||||
assert(size_a1 + size_a2 + 2 == a.size()); |
||||
assert(size_b1 + size_b2 + 2 == b.size()); |
||||
|
||||
Slice a1 = Slice(a.data() + 2, size_a1); |
||||
Slice b1 = Slice(b.data() + 2, size_b1); |
||||
Slice a2 = Slice(a.data() + 2 + size_a1, size_a2); |
||||
Slice b2 = Slice(b.data() + 2 + size_b1, size_b2); |
||||
|
||||
if (a1 != b1) { |
||||
return a1.compare(b1); |
||||
} |
||||
return a2.compare(b2); |
||||
} |
||||
virtual void FindShortestSeparator(std::string* start, |
||||
const Slice& limit) const {} |
||||
|
||||
virtual void FindShortSuccessor(std::string* key) const {} |
||||
}; |
||||
} // namespace
|
||||
|
||||
class ComparatorDBTest { |
||||
private: |
||||
std::string dbname_; |
||||
Env* env_; |
||||
DB* db_; |
||||
Options last_options_; |
||||
std::unique_ptr<const Comparator> comparator_guard; |
||||
|
||||
public: |
||||
ComparatorDBTest() : env_(Env::Default()), db_(nullptr) { |
||||
comparator = BytewiseComparator(); |
||||
dbname_ = test::TmpDir() + "/comparator_db_test"; |
||||
ASSERT_OK(DestroyDB(dbname_, last_options_)); |
||||
} |
||||
|
||||
~ComparatorDBTest() { |
||||
delete db_; |
||||
ASSERT_OK(DestroyDB(dbname_, last_options_)); |
||||
comparator = BytewiseComparator(); |
||||
} |
||||
|
||||
DB* GetDB() { return db_; } |
||||
|
||||
void SetOwnedComparator(const Comparator* cmp) { |
||||
comparator_guard.reset(cmp); |
||||
comparator = cmp; |
||||
last_options_.comparator = cmp; |
||||
} |
||||
|
||||
// Return the current option configuration.
|
||||
Options* GetOptions() { return &last_options_; } |
||||
|
||||
void DestroyAndReopen() { |
||||
// Destroy using last options
|
||||
Destroy(); |
||||
ASSERT_OK(TryReopen()); |
||||
} |
||||
|
||||
void Destroy() { |
||||
delete db_; |
||||
db_ = nullptr; |
||||
ASSERT_OK(DestroyDB(dbname_, last_options_)); |
||||
} |
||||
|
||||
Status TryReopen() { |
||||
delete db_; |
||||
db_ = nullptr; |
||||
last_options_.create_if_missing = true; |
||||
|
||||
return DB::Open(last_options_, dbname_, &db_); |
||||
} |
||||
}; |
||||
|
||||
TEST(ComparatorDBTest, Bytewise) { |
||||
for (int rand_seed = 301; rand_seed < 306; rand_seed++) { |
||||
DestroyAndReopen(); |
||||
Random rnd(rand_seed); |
||||
DoRandomIteraratorTest(GetDB(), |
||||
{"a", "b", "c", "d", "e", "f", "g", "h", "i"}, &rnd, |
||||
8, 100, 3); |
||||
} |
||||
} |
||||
|
||||
TEST(ComparatorDBTest, SimpleSuffixReverseComparator) { |
||||
SetOwnedComparator(new test::SimpleSuffixReverseComparator()); |
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { |
||||
Options* opt = GetOptions(); |
||||
opt->comparator = comparator; |
||||
DestroyAndReopen(); |
||||
Random rnd(rnd_seed); |
||||
|
||||
std::vector<std::string> source_strings; |
||||
std::vector<std::string> source_prefixes; |
||||
// Randomly generate 5 prefixes
|
||||
for (int i = 0; i < 5; i++) { |
||||
source_prefixes.push_back(test::RandomHumanReadableString(&rnd, 8)); |
||||
} |
||||
for (int j = 0; j < 20; j++) { |
||||
int prefix_index = rnd.Uniform(static_cast<int>(source_prefixes.size())); |
||||
std::string key = source_prefixes[prefix_index] + |
||||
test::RandomHumanReadableString(&rnd, rnd.Uniform(8)); |
||||
source_strings.push_back(key); |
||||
} |
||||
|
||||
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 30, 600, 66); |
||||
} |
||||
} |
||||
|
||||
TEST(ComparatorDBTest, Uint64Comparator) { |
||||
SetOwnedComparator(test::Uint64Comparator()); |
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { |
||||
Options* opt = GetOptions(); |
||||
opt->comparator = comparator; |
||||
DestroyAndReopen(); |
||||
Random rnd(rnd_seed); |
||||
Random64 rnd64(rnd_seed); |
||||
|
||||
std::vector<std::string> source_strings; |
||||
// Randomly generate source keys
|
||||
for (int i = 0; i < 100; i++) { |
||||
uint64_t r = rnd64.Next(); |
||||
std::string str; |
||||
str.resize(8); |
||||
memcpy(&str[0], static_cast<void*>(&r), 8); |
||||
source_strings.push_back(str); |
||||
} |
||||
|
||||
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66); |
||||
} |
||||
} |
||||
|
||||
TEST(ComparatorDBTest, DoubleComparator) { |
||||
SetOwnedComparator(new DoubleComparator()); |
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { |
||||
Options* opt = GetOptions(); |
||||
opt->comparator = comparator; |
||||
DestroyAndReopen(); |
||||
Random rnd(rnd_seed); |
||||
|
||||
std::vector<std::string> source_strings; |
||||
// Randomly generate source keys
|
||||
for (int i = 0; i < 100; i++) { |
||||
uint32_t r = rnd.Next(); |
||||
uint32_t divide_order = rnd.Uniform(8); |
||||
double to_divide = 1.0; |
||||
for (uint32_t j = 0; j < divide_order; j++) { |
||||
to_divide *= 10.0; |
||||
} |
||||
source_strings.push_back(ToString(r / to_divide)); |
||||
} |
||||
|
||||
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66); |
||||
} |
||||
} |
||||
|
||||
TEST(ComparatorDBTest, HashComparator) { |
||||
SetOwnedComparator(new HashComparator()); |
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { |
||||
Options* opt = GetOptions(); |
||||
opt->comparator = comparator; |
||||
DestroyAndReopen(); |
||||
Random rnd(rnd_seed); |
||||
|
||||
std::vector<std::string> source_strings; |
||||
// Randomly generate source keys
|
||||
for (int i = 0; i < 100; i++) { |
||||
source_strings.push_back(test::RandomKey(&rnd, 8)); |
||||
} |
||||
|
||||
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66); |
||||
} |
||||
} |
||||
|
||||
TEST(ComparatorDBTest, TwoStrComparator) { |
||||
SetOwnedComparator(new TwoStrComparator()); |
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) { |
||||
Options* opt = GetOptions(); |
||||
opt->comparator = comparator; |
||||
DestroyAndReopen(); |
||||
Random rnd(rnd_seed); |
||||
|
||||
std::vector<std::string> source_strings; |
||||
// Randomly generate source keys
|
||||
for (int i = 0; i < 100; i++) { |
||||
std::string str; |
||||
uint32_t size1 = rnd.Uniform(8); |
||||
uint32_t size2 = rnd.Uniform(8); |
||||
str.append(1, static_cast<char>(size1)); |
||||
str.append(1, static_cast<char>(size2)); |
||||
str.append(test::RandomKey(&rnd, size1)); |
||||
str.append(test::RandomKey(&rnd, size2)); |
||||
source_strings.push_back(str); |
||||
} |
||||
|
||||
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66); |
||||
} |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } |
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,742 @@ |
||||
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright 2014 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
// This test uses a custom Env to keep track of the state of a filesystem as of
|
||||
// the last "sync". It then checks for data loss errors by purposely dropping
|
||||
// file data (or entire files) not protected by a "sync".
|
||||
|
||||
#include <map> |
||||
#include <set> |
||||
#include "db/db_impl.h" |
||||
#include "db/filename.h" |
||||
#include "db/log_format.h" |
||||
#include "db/version_set.h" |
||||
#include "rocksdb/cache.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/table.h" |
||||
#include "rocksdb/write_batch.h" |
||||
#include "util/logging.h" |
||||
#include "util/mock_env.h" |
||||
#include "util/mutexlock.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
static const int kValueSize = 1000; |
||||
static const int kMaxNumValues = 2000; |
||||
static const size_t kNumIterations = 3; |
||||
|
||||
class TestWritableFile; |
||||
class FaultInjectionTestEnv; |
||||
|
||||
namespace { |
||||
|
||||
// Assume a filename, and not a directory name like "/foo/bar/"
|
||||
static std::string GetDirName(const std::string filename) { |
||||
size_t found = filename.find_last_of("/\\"); |
||||
if (found == std::string::npos) { |
||||
return ""; |
||||
} else { |
||||
return filename.substr(0, found); |
||||
} |
||||
} |
||||
|
||||
// Trim the tailing "/" in the end of `str`
|
||||
static std::string TrimDirname(const std::string& str) { |
||||
size_t found = str.find_last_not_of("/"); |
||||
if (found == std::string::npos) { |
||||
return str; |
||||
} |
||||
return str.substr(0, found + 1); |
||||
} |
||||
|
||||
// Return pair <parent directory name, file name> of a full path.
|
||||
static std::pair<std::string, std::string> GetDirAndName( |
||||
const std::string& name) { |
||||
std::string dirname = GetDirName(name); |
||||
std::string fname = name.substr(dirname.size() + 1); |
||||
return std::make_pair(dirname, fname); |
||||
} |
||||
|
||||
// A basic file truncation function suitable for this test.
|
||||
Status Truncate(Env* env, const std::string& filename, uint64_t length) { |
||||
unique_ptr<SequentialFile> orig_file; |
||||
const EnvOptions options; |
||||
Status s = env->NewSequentialFile(filename, &orig_file, options); |
||||
if (!s.ok()) { |
||||
fprintf(stderr, "Cannot truncate file %s: %s\n", filename.c_str(), |
||||
s.ToString().c_str()); |
||||
return s; |
||||
} |
||||
|
||||
char* scratch = new char[length]; |
||||
rocksdb::Slice result; |
||||
s = orig_file->Read(length, &result, scratch); |
||||
if (s.ok()) { |
||||
std::string tmp_name = GetDirName(filename) + "/truncate.tmp"; |
||||
unique_ptr<WritableFile> tmp_file; |
||||
s = env->NewWritableFile(tmp_name, &tmp_file, options); |
||||
if (s.ok()) { |
||||
s = tmp_file->Append(result); |
||||
if (s.ok()) { |
||||
s = env->RenameFile(tmp_name, filename); |
||||
} else { |
||||
fprintf(stderr, "Cannot rename file %s to %s: %s\n", tmp_name.c_str(), |
||||
filename.c_str(), s.ToString().c_str()); |
||||
env->DeleteFile(tmp_name); |
||||
} |
||||
} |
||||
} |
||||
if (!s.ok()) { |
||||
fprintf(stderr, "Cannot truncate file %s: %s\n", filename.c_str(), |
||||
s.ToString().c_str()); |
||||
} |
||||
|
||||
delete[] scratch; |
||||
|
||||
return s; |
||||
} |
||||
|
||||
struct FileState { |
||||
std::string filename_; |
||||
ssize_t pos_; |
||||
ssize_t pos_at_last_sync_; |
||||
ssize_t pos_at_last_flush_; |
||||
|
||||
explicit FileState(const std::string& filename) |
||||
: filename_(filename), |
||||
pos_(-1), |
||||
pos_at_last_sync_(-1), |
||||
pos_at_last_flush_(-1) { } |
||||
|
||||
FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {} |
||||
|
||||
bool IsFullySynced() const { return pos_ <= 0 || pos_ == pos_at_last_sync_; } |
||||
|
||||
Status DropUnsyncedData(Env* env) const; |
||||
|
||||
Status DropRandomUnsyncedData(Env* env, Random* rand) const; |
||||
}; |
||||
|
||||
} // anonymous namespace
|
||||
|
||||
// A wrapper around WritableFile which informs another Env whenever this file
|
||||
// is written to or sync'ed.
|
||||
class TestWritableFile : public WritableFile { |
||||
public: |
||||
explicit TestWritableFile(const std::string& fname, |
||||
unique_ptr<WritableFile>&& f, |
||||
FaultInjectionTestEnv* env); |
||||
virtual ~TestWritableFile(); |
||||
virtual Status Append(const Slice& data); |
||||
virtual Status Close(); |
||||
virtual Status Flush(); |
||||
virtual Status Sync(); |
||||
|
||||
private: |
||||
FileState state_; |
||||
unique_ptr<WritableFile> target_; |
||||
bool writable_file_opened_; |
||||
FaultInjectionTestEnv* env_; |
||||
}; |
||||
|
||||
class TestDirectory : public Directory { |
||||
public: |
||||
explicit TestDirectory(FaultInjectionTestEnv* env, std::string dirname, |
||||
Directory* dir) |
||||
: env_(env), dirname_(dirname), dir_(dir) {} |
||||
~TestDirectory() {} |
||||
|
||||
virtual Status Fsync() override; |
||||
|
||||
private: |
||||
FaultInjectionTestEnv* env_; |
||||
std::string dirname_; |
||||
unique_ptr<Directory> dir_; |
||||
}; |
||||
|
||||
class FaultInjectionTestEnv : public EnvWrapper { |
||||
public: |
||||
explicit FaultInjectionTestEnv(Env* base) |
||||
: EnvWrapper(base), |
||||
filesystem_active_(true) {} |
||||
virtual ~FaultInjectionTestEnv() { } |
||||
|
||||
Status NewDirectory(const std::string& name, |
||||
unique_ptr<Directory>* result) override { |
||||
unique_ptr<Directory> r; |
||||
Status s = target()->NewDirectory(name, &r); |
||||
ASSERT_OK(s); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
result->reset(new TestDirectory(this, TrimDirname(name), r.release())); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status NewWritableFile(const std::string& fname, |
||||
unique_ptr<WritableFile>* result, |
||||
const EnvOptions& soptions) { |
||||
Status s = target()->NewWritableFile(fname, result, soptions); |
||||
if (s.ok()) { |
||||
result->reset(new TestWritableFile(fname, std::move(*result), this)); |
||||
// WritableFile doesn't append to files, so if the same file is opened
|
||||
// again then it will be truncated - so forget our saved state.
|
||||
UntrackFile(fname); |
||||
MutexLock l(&mutex_); |
||||
open_files_.insert(fname); |
||||
auto dir_and_name = GetDirAndName(fname); |
||||
auto& list = dir_to_new_files_since_last_sync_[dir_and_name.first]; |
||||
list.insert(dir_and_name.second); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
virtual Status DeleteFile(const std::string& f) { |
||||
Status s = EnvWrapper::DeleteFile(f); |
||||
if (!s.ok()) { |
||||
fprintf(stderr, "Cannot delete file %s: %s\n", f.c_str(), |
||||
s.ToString().c_str()); |
||||
} |
||||
ASSERT_OK(s); |
||||
if (s.ok()) { |
||||
UntrackFile(f); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
virtual Status RenameFile(const std::string& s, const std::string& t) { |
||||
Status ret = EnvWrapper::RenameFile(s, t); |
||||
|
||||
if (ret.ok()) { |
||||
MutexLock l(&mutex_); |
||||
if (db_file_state_.find(s) != db_file_state_.end()) { |
||||
db_file_state_[t] = db_file_state_[s]; |
||||
db_file_state_.erase(s); |
||||
} |
||||
|
||||
auto sdn = GetDirAndName(s); |
||||
auto tdn = GetDirAndName(t); |
||||
if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) { |
||||
auto& tlist = dir_to_new_files_since_last_sync_[tdn.first]; |
||||
assert(tlist.find(tdn.second) == tlist.end()); |
||||
tlist.insert(tdn.second); |
||||
} |
||||
} |
||||
|
||||
return ret; |
||||
} |
||||
|
||||
void WritableFileClosed(const FileState& state) { |
||||
MutexLock l(&mutex_); |
||||
if (open_files_.find(state.filename_) != open_files_.end()) { |
||||
db_file_state_[state.filename_] = state; |
||||
open_files_.erase(state.filename_); |
||||
} |
||||
} |
||||
|
||||
// For every file that is not fully synced, make a call to `func` with
|
||||
// FileState of the file as the parameter.
|
||||
Status DropFileData(std::function<Status(Env*, FileState)> func) { |
||||
Status s; |
||||
MutexLock l(&mutex_); |
||||
for (std::map<std::string, FileState>::const_iterator it = |
||||
db_file_state_.begin(); |
||||
s.ok() && it != db_file_state_.end(); ++it) { |
||||
const FileState& state = it->second; |
||||
if (!state.IsFullySynced()) { |
||||
s = func(target(), state); |
||||
} |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status DropUnsyncedFileData() { |
||||
return DropFileData([&](Env* env, const FileState& state) { |
||||
return state.DropUnsyncedData(env); |
||||
}); |
||||
} |
||||
|
||||
Status DropRandomUnsyncedFileData(Random* rnd) { |
||||
return DropFileData([&](Env* env, const FileState& state) { |
||||
return state.DropRandomUnsyncedData(env, rnd); |
||||
}); |
||||
} |
||||
|
||||
Status DeleteFilesCreatedAfterLastDirSync() { |
||||
// Because DeleteFile access this container make a copy to avoid deadlock
|
||||
std::map<std::string, std::set<std::string>> map_copy; |
||||
{ |
||||
MutexLock l(&mutex_); |
||||
map_copy.insert(dir_to_new_files_since_last_sync_.begin(), |
||||
dir_to_new_files_since_last_sync_.end()); |
||||
} |
||||
|
||||
for (auto& pair : map_copy) { |
||||
for (std::string name : pair.second) { |
||||
Status s = DeleteFile(pair.first + "/" + name); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
} |
||||
} |
||||
return Status::OK(); |
||||
} |
||||
void ResetState() { |
||||
MutexLock l(&mutex_); |
||||
db_file_state_.clear(); |
||||
dir_to_new_files_since_last_sync_.clear(); |
||||
SetFilesystemActiveNoLock(true); |
||||
} |
||||
|
||||
void UntrackFile(const std::string& f) { |
||||
MutexLock l(&mutex_); |
||||
auto dir_and_name = GetDirAndName(f); |
||||
dir_to_new_files_since_last_sync_[dir_and_name.first].erase( |
||||
dir_and_name.second); |
||||
db_file_state_.erase(f); |
||||
open_files_.erase(f); |
||||
} |
||||
|
||||
void SyncDir(const std::string& dirname) { |
||||
MutexLock l(&mutex_); |
||||
dir_to_new_files_since_last_sync_.erase(dirname); |
||||
} |
||||
|
||||
// Setting the filesystem to inactive is the test equivalent to simulating a
|
||||
// system reset. Setting to inactive will freeze our saved filesystem state so
|
||||
// that it will stop being recorded. It can then be reset back to the state at
|
||||
// the time of the reset.
|
||||
bool IsFilesystemActive() { |
||||
MutexLock l(&mutex_); |
||||
return filesystem_active_; |
||||
} |
||||
void SetFilesystemActiveNoLock(bool active) { filesystem_active_ = active; } |
||||
void SetFilesystemActive(bool active) { |
||||
MutexLock l(&mutex_); |
||||
SetFilesystemActiveNoLock(active); |
||||
} |
||||
void AssertNoOpenFile() { ASSERT_TRUE(open_files_.empty()); } |
||||
|
||||
private: |
||||
port::Mutex mutex_; |
||||
std::map<std::string, FileState> db_file_state_; |
||||
std::set<std::string> open_files_; |
||||
std::unordered_map<std::string, std::set<std::string>> |
||||
dir_to_new_files_since_last_sync_; |
||||
bool filesystem_active_; // Record flushes, syncs, writes
|
||||
}; |
||||
|
||||
Status FileState::DropUnsyncedData(Env* env) const { |
||||
ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; |
||||
return Truncate(env, filename_, sync_pos); |
||||
} |
||||
|
||||
Status FileState::DropRandomUnsyncedData(Env* env, Random* rand) const { |
||||
ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; |
||||
assert(pos_ >= sync_pos); |
||||
int range = static_cast<int>(pos_ - sync_pos); |
||||
uint64_t truncated_size = |
||||
static_cast<uint64_t>(sync_pos) + rand->Uniform(range); |
||||
return Truncate(env, filename_, truncated_size); |
||||
} |
||||
|
||||
Status TestDirectory::Fsync() { |
||||
env_->SyncDir(dirname_); |
||||
return dir_->Fsync(); |
||||
} |
||||
|
||||
TestWritableFile::TestWritableFile(const std::string& fname, |
||||
unique_ptr<WritableFile>&& f, |
||||
FaultInjectionTestEnv* env) |
||||
: state_(fname), |
||||
target_(std::move(f)), |
||||
writable_file_opened_(true), |
||||
env_(env) { |
||||
assert(target_ != nullptr); |
||||
state_.pos_ = 0; |
||||
} |
||||
|
||||
TestWritableFile::~TestWritableFile() { |
||||
if (writable_file_opened_) { |
||||
Close(); |
||||
} |
||||
} |
||||
|
||||
Status TestWritableFile::Append(const Slice& data) { |
||||
Status s = target_->Append(data); |
||||
if (s.ok() && env_->IsFilesystemActive()) { |
||||
state_.pos_ += data.size(); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status TestWritableFile::Close() { |
||||
writable_file_opened_ = false; |
||||
Status s = target_->Close(); |
||||
if (s.ok()) { |
||||
env_->WritableFileClosed(state_); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status TestWritableFile::Flush() { |
||||
Status s = target_->Flush(); |
||||
if (s.ok() && env_->IsFilesystemActive()) { |
||||
state_.pos_at_last_flush_ = state_.pos_; |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status TestWritableFile::Sync() { |
||||
if (!env_->IsFilesystemActive()) { |
||||
return Status::OK(); |
||||
} |
||||
// No need to actual sync.
|
||||
state_.pos_at_last_sync_ = state_.pos_; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
class FaultInjectionTest { |
||||
protected: |
||||
enum OptionConfig { |
||||
kDefault, |
||||
kDifferentDataDir, |
||||
kWalDir, |
||||
kSyncWal, |
||||
kWalDirSyncWal, |
||||
kMultiLevels, |
||||
kEnd, |
||||
}; |
||||
int option_config_; |
||||
// When need to make sure data is persistent, sync WAL
|
||||
bool sync_use_wal_; |
||||
// When need to make sure data is persistent, call DB::CompactRange()
|
||||
bool sync_use_compact_; |
||||
|
||||
protected: |
||||
public: |
||||
enum ExpectedVerifResult { kValExpectFound, kValExpectNoError }; |
||||
enum ResetMethod { |
||||
kResetDropUnsyncedData, |
||||
kResetDropRandomUnsyncedData, |
||||
kResetDeleteUnsyncedFiles, |
||||
kResetDropAndDeleteUnsynced |
||||
}; |
||||
|
||||
std::unique_ptr<Env> base_env_; |
||||
FaultInjectionTestEnv* env_; |
||||
std::string dbname_; |
||||
shared_ptr<Cache> tiny_cache_; |
||||
Options options_; |
||||
DB* db_; |
||||
|
||||
FaultInjectionTest() |
||||
: option_config_(kDefault), |
||||
sync_use_wal_(false), |
||||
sync_use_compact_(true), |
||||
base_env_(nullptr), |
||||
env_(NULL), |
||||
db_(NULL) { |
||||
NewDB(); |
||||
} |
||||
|
||||
~FaultInjectionTest() { ASSERT_OK(TearDown()); } |
||||
|
||||
bool ChangeOptions() { |
||||
option_config_++; |
||||
if (option_config_ >= kEnd) { |
||||
return false; |
||||
} else { |
||||
if (option_config_ == kMultiLevels) { |
||||
base_env_.reset(new MockEnv(Env::Default())); |
||||
} |
||||
return true; |
||||
} |
||||
} |
||||
|
||||
// Return the current option configuration.
|
||||
Options CurrentOptions() { |
||||
sync_use_wal_ = false; |
||||
sync_use_compact_ = true; |
||||
Options options; |
||||
switch (option_config_) { |
||||
case kWalDir: |
||||
options.wal_dir = test::TmpDir(env_) + "/fault_test_wal"; |
||||
break; |
||||
case kDifferentDataDir: |
||||
options.db_paths.emplace_back(test::TmpDir(env_) + "/fault_test_data", |
||||
1000000U); |
||||
break; |
||||
case kSyncWal: |
||||
sync_use_wal_ = true; |
||||
sync_use_compact_ = false; |
||||
break; |
||||
case kWalDirSyncWal: |
||||
options.wal_dir = test::TmpDir(env_) + "/fault_test_wal"; |
||||
sync_use_wal_ = true; |
||||
sync_use_compact_ = false; |
||||
break; |
||||
case kMultiLevels: |
||||
options.write_buffer_size = 64 * 1024; |
||||
options.target_file_size_base = 64 * 1024; |
||||
options.level0_file_num_compaction_trigger = 2; |
||||
options.level0_slowdown_writes_trigger = 2; |
||||
options.level0_stop_writes_trigger = 4; |
||||
options.max_bytes_for_level_base = 128 * 1024; |
||||
options.max_write_buffer_number = 2; |
||||
options.max_background_compactions = 8; |
||||
options.max_background_flushes = 8; |
||||
sync_use_wal_ = true; |
||||
sync_use_compact_ = false; |
||||
break; |
||||
default: |
||||
break; |
||||
} |
||||
return options; |
||||
} |
||||
|
||||
Status NewDB() { |
||||
assert(db_ == NULL); |
||||
assert(tiny_cache_ == nullptr); |
||||
assert(env_ == NULL); |
||||
|
||||
env_ = |
||||
new FaultInjectionTestEnv(base_env_ ? base_env_.get() : Env::Default()); |
||||
|
||||
options_ = CurrentOptions(); |
||||
options_.env = env_; |
||||
options_.paranoid_checks = true; |
||||
|
||||
BlockBasedTableOptions table_options; |
||||
tiny_cache_ = NewLRUCache(100); |
||||
table_options.block_cache = tiny_cache_; |
||||
options_.table_factory.reset(NewBlockBasedTableFactory(table_options)); |
||||
|
||||
dbname_ = test::TmpDir() + "/fault_test"; |
||||
|
||||
ASSERT_OK(DestroyDB(dbname_, options_)); |
||||
|
||||
options_.create_if_missing = true; |
||||
Status s = OpenDB(); |
||||
options_.create_if_missing = false; |
||||
return s; |
||||
} |
||||
|
||||
Status SetUp() { |
||||
Status s = TearDown(); |
||||
if (s.ok()) { |
||||
s = NewDB(); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status TearDown() { |
||||
CloseDB(); |
||||
|
||||
Status s = DestroyDB(dbname_, options_); |
||||
|
||||
delete env_; |
||||
env_ = NULL; |
||||
|
||||
tiny_cache_.reset(); |
||||
|
||||
return s; |
||||
} |
||||
|
||||
void Build(const WriteOptions& write_options, int start_idx, int num_vals) { |
||||
std::string key_space, value_space; |
||||
WriteBatch batch; |
||||
for (int i = start_idx; i < start_idx + num_vals; i++) { |
||||
Slice key = Key(i, &key_space); |
||||
batch.Clear(); |
||||
batch.Put(key, Value(i, &value_space)); |
||||
ASSERT_OK(db_->Write(write_options, &batch)); |
||||
} |
||||
} |
||||
|
||||
Status ReadValue(int i, std::string* val) const { |
||||
std::string key_space, value_space; |
||||
Slice key = Key(i, &key_space); |
||||
Value(i, &value_space); |
||||
ReadOptions options; |
||||
return db_->Get(options, key, val); |
||||
} |
||||
|
||||
Status Verify(int start_idx, int num_vals, |
||||
ExpectedVerifResult expected) const { |
||||
std::string val; |
||||
std::string value_space; |
||||
Status s; |
||||
for (int i = start_idx; i < start_idx + num_vals && s.ok(); i++) { |
||||
Value(i, &value_space); |
||||
s = ReadValue(i, &val); |
||||
if (s.ok()) { |
||||
ASSERT_EQ(value_space, val); |
||||
} |
||||
if (expected == kValExpectFound) { |
||||
if (!s.ok()) { |
||||
fprintf(stderr, "Error when read %dth record (expect found): %s\n", i, |
||||
s.ToString().c_str()); |
||||
return s; |
||||
} |
||||
} else if (!s.ok() && !s.IsNotFound()) { |
||||
fprintf(stderr, "Error when read %dth record: %s\n", i, |
||||
s.ToString().c_str()); |
||||
return s; |
||||
} |
||||
} |
||||
return Status::OK(); |
||||
} |
||||
|
||||
// Return the ith key
|
||||
Slice Key(int i, std::string* storage) const { |
||||
char buf[100]; |
||||
snprintf(buf, sizeof(buf), "%016d", i); |
||||
storage->assign(buf, strlen(buf)); |
||||
return Slice(*storage); |
||||
} |
||||
|
||||
// Return the value to associate with the specified key
|
||||
Slice Value(int k, std::string* storage) const { |
||||
Random r(k); |
||||
return test::RandomString(&r, kValueSize, storage); |
||||
} |
||||
|
||||
Status OpenDB() { |
||||
delete db_; |
||||
db_ = NULL; |
||||
env_->ResetState(); |
||||
return DB::Open(options_, dbname_, &db_); |
||||
} |
||||
|
||||
void CloseDB() { |
||||
delete db_; |
||||
db_ = NULL; |
||||
} |
||||
|
||||
void DeleteAllData() { |
||||
Iterator* iter = db_->NewIterator(ReadOptions()); |
||||
WriteOptions options; |
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { |
||||
ASSERT_OK(db_->Delete(WriteOptions(), iter->key())); |
||||
} |
||||
|
||||
delete iter; |
||||
|
||||
FlushOptions flush_options; |
||||
flush_options.wait = true; |
||||
db_->Flush(flush_options); |
||||
} |
||||
|
||||
// rnd cannot be null for kResetDropRandomUnsyncedData
|
||||
void ResetDBState(ResetMethod reset_method, Random* rnd = nullptr) { |
||||
env_->AssertNoOpenFile(); |
||||
switch (reset_method) { |
||||
case kResetDropUnsyncedData: |
||||
ASSERT_OK(env_->DropUnsyncedFileData()); |
||||
break; |
||||
case kResetDropRandomUnsyncedData: |
||||
ASSERT_OK(env_->DropRandomUnsyncedFileData(rnd)); |
||||
break; |
||||
case kResetDeleteUnsyncedFiles: |
||||
ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync()); |
||||
break; |
||||
case kResetDropAndDeleteUnsynced: |
||||
ASSERT_OK(env_->DropUnsyncedFileData()); |
||||
ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync()); |
||||
break; |
||||
default: |
||||
assert(false); |
||||
} |
||||
} |
||||
|
||||
void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) { |
||||
DeleteAllData(); |
||||
|
||||
WriteOptions write_options; |
||||
write_options.sync = sync_use_wal_; |
||||
|
||||
Build(write_options, 0, num_pre_sync); |
||||
if (sync_use_compact_) { |
||||
db_->CompactRange(nullptr, nullptr); |
||||
} |
||||
write_options.sync = false; |
||||
Build(write_options, num_pre_sync, num_post_sync); |
||||
} |
||||
|
||||
void PartialCompactTestReopenWithFault(ResetMethod reset_method, |
||||
int num_pre_sync, int num_post_sync, |
||||
Random* rnd = nullptr) { |
||||
env_->SetFilesystemActive(false); |
||||
CloseDB(); |
||||
ResetDBState(reset_method, rnd); |
||||
ASSERT_OK(OpenDB()); |
||||
ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::kValExpectFound)); |
||||
ASSERT_OK(Verify(num_pre_sync, num_post_sync, |
||||
FaultInjectionTest::kValExpectNoError)); |
||||
} |
||||
|
||||
void NoWriteTestPreFault() { |
||||
} |
||||
|
||||
void NoWriteTestReopenWithFault(ResetMethod reset_method) { |
||||
CloseDB(); |
||||
ResetDBState(reset_method); |
||||
ASSERT_OK(OpenDB()); |
||||
} |
||||
}; |
||||
|
||||
TEST(FaultInjectionTest, FaultTest) { |
||||
do { |
||||
Random rnd(301); |
||||
ASSERT_OK(SetUp()); |
||||
|
||||
for (size_t idx = 0; idx < kNumIterations; idx++) { |
||||
int num_pre_sync = rnd.Uniform(kMaxNumValues); |
||||
int num_post_sync = rnd.Uniform(kMaxNumValues); |
||||
|
||||
PartialCompactTestPreFault(num_pre_sync, num_post_sync); |
||||
PartialCompactTestReopenWithFault(kResetDropUnsyncedData, num_pre_sync, |
||||
num_post_sync); |
||||
NoWriteTestPreFault(); |
||||
NoWriteTestReopenWithFault(kResetDropUnsyncedData); |
||||
|
||||
PartialCompactTestPreFault(num_pre_sync, num_post_sync); |
||||
PartialCompactTestReopenWithFault(kResetDropRandomUnsyncedData, |
||||
num_pre_sync, num_post_sync, &rnd); |
||||
NoWriteTestPreFault(); |
||||
NoWriteTestReopenWithFault(kResetDropUnsyncedData); |
||||
|
||||
// Setting a separate data path won't pass the test as we don't sync
|
||||
// it after creating new files,
|
||||
PartialCompactTestPreFault(num_pre_sync, num_post_sync); |
||||
PartialCompactTestReopenWithFault(kResetDropAndDeleteUnsynced, |
||||
num_pre_sync, num_post_sync); |
||||
NoWriteTestPreFault(); |
||||
NoWriteTestReopenWithFault(kResetDropAndDeleteUnsynced); |
||||
|
||||
PartialCompactTestPreFault(num_pre_sync, num_post_sync); |
||||
// No new files created so we expect all values since no files will be
|
||||
// dropped.
|
||||
PartialCompactTestReopenWithFault(kResetDeleteUnsyncedFiles, num_pre_sync, |
||||
num_post_sync); |
||||
NoWriteTestPreFault(); |
||||
NoWriteTestReopenWithFault(kResetDeleteUnsyncedFiles); |
||||
} |
||||
} while (ChangeOptions()); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
return rocksdb::test::RunAllTests(); |
||||
} |
||||
@ -0,0 +1,221 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/flush_job.h" |
||||
|
||||
#ifndef __STDC_FORMAT_MACROS |
||||
#define __STDC_FORMAT_MACROS |
||||
#endif |
||||
|
||||
#include <inttypes.h> |
||||
#include <algorithm> |
||||
#include <vector> |
||||
|
||||
#include "db/builder.h" |
||||
#include "db/db_iter.h" |
||||
#include "db/dbformat.h" |
||||
#include "db/filename.h" |
||||
#include "db/log_reader.h" |
||||
#include "db/log_writer.h" |
||||
#include "db/memtable.h" |
||||
#include "db/memtable_list.h" |
||||
#include "db/merge_context.h" |
||||
#include "db/version_set.h" |
||||
#include "port/port.h" |
||||
#include "port/likely.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/statistics.h" |
||||
#include "rocksdb/status.h" |
||||
#include "rocksdb/table.h" |
||||
#include "table/block.h" |
||||
#include "table/block_based_table_factory.h" |
||||
#include "table/merger.h" |
||||
#include "table/table_builder.h" |
||||
#include "table/two_level_iterator.h" |
||||
#include "util/coding.h" |
||||
#include "util/file_util.h" |
||||
#include "util/logging.h" |
||||
#include "util/log_buffer.h" |
||||
#include "util/mutexlock.h" |
||||
#include "util/perf_context_imp.h" |
||||
#include "util/iostats_context_imp.h" |
||||
#include "util/stop_watch.h" |
||||
#include "util/sync_point.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd, |
||||
const DBOptions& db_options, |
||||
const MutableCFOptions& mutable_cf_options, |
||||
const EnvOptions& env_options, VersionSet* versions, |
||||
InstrumentedMutex* db_mutex, |
||||
std::atomic<bool>* shutting_down, |
||||
SequenceNumber newest_snapshot, JobContext* job_context, |
||||
LogBuffer* log_buffer, Directory* db_directory, |
||||
Directory* output_file_directory, |
||||
CompressionType output_compression, Statistics* stats) |
||||
: dbname_(dbname), |
||||
cfd_(cfd), |
||||
db_options_(db_options), |
||||
mutable_cf_options_(mutable_cf_options), |
||||
env_options_(env_options), |
||||
versions_(versions), |
||||
db_mutex_(db_mutex), |
||||
shutting_down_(shutting_down), |
||||
newest_snapshot_(newest_snapshot), |
||||
job_context_(job_context), |
||||
log_buffer_(log_buffer), |
||||
db_directory_(db_directory), |
||||
output_file_directory_(output_file_directory), |
||||
output_compression_(output_compression), |
||||
stats_(stats) {} |
||||
|
||||
Status FlushJob::Run(uint64_t* file_number) { |
||||
// Save the contents of the earliest memtable as a new Table
|
||||
uint64_t fn; |
||||
autovector<MemTable*> mems; |
||||
cfd_->imm()->PickMemtablesToFlush(&mems); |
||||
if (mems.empty()) { |
||||
LogToBuffer(log_buffer_, "[%s] Nothing in memtable to flush", |
||||
cfd_->GetName().c_str()); |
||||
return Status::OK(); |
||||
} |
||||
|
||||
// entries mems are (implicitly) sorted in ascending order by their created
|
||||
// time. We will use the first memtable's `edit` to keep the meta info for
|
||||
// this flush.
|
||||
MemTable* m = mems[0]; |
||||
VersionEdit* edit = m->GetEdits(); |
||||
edit->SetPrevLogNumber(0); |
||||
// SetLogNumber(log_num) indicates logs with number smaller than log_num
|
||||
// will no longer be picked up for recovery.
|
||||
edit->SetLogNumber(mems.back()->GetNextLogNumber()); |
||||
edit->SetColumnFamily(cfd_->GetID()); |
||||
|
||||
// This will release and re-acquire the mutex.
|
||||
Status s = WriteLevel0Table(mems, edit, &fn); |
||||
|
||||
if (s.ok() && |
||||
(shutting_down_->load(std::memory_order_acquire) || cfd_->IsDropped())) { |
||||
s = Status::ShutdownInProgress( |
||||
"Database shutdown or Column family drop during flush"); |
||||
} |
||||
|
||||
if (!s.ok()) { |
||||
cfd_->imm()->RollbackMemtableFlush(mems, fn); |
||||
} else { |
||||
// Replace immutable memtable with the generated Table
|
||||
s = cfd_->imm()->InstallMemtableFlushResults( |
||||
cfd_, mutable_cf_options_, mems, versions_, db_mutex_, fn, |
||||
&job_context_->memtables_to_free, db_directory_, log_buffer_); |
||||
} |
||||
|
||||
if (s.ok() && file_number != nullptr) { |
||||
*file_number = fn; |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems, |
||||
VersionEdit* edit, uint64_t* filenumber) { |
||||
db_mutex_->AssertHeld(); |
||||
const uint64_t start_micros = db_options_.env->NowMicros(); |
||||
FileMetaData meta; |
||||
// path 0 for level 0 file.
|
||||
meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0); |
||||
*filenumber = meta.fd.GetNumber(); |
||||
|
||||
const SequenceNumber earliest_seqno_in_memtable = |
||||
mems[0]->GetFirstSequenceNumber(); |
||||
Version* base = cfd_->current(); |
||||
base->Ref(); // it is likely that we do not need this reference
|
||||
Status s; |
||||
{ |
||||
db_mutex_->Unlock(); |
||||
if (log_buffer_) { |
||||
log_buffer_->FlushBufferToLog(); |
||||
} |
||||
std::vector<Iterator*> memtables; |
||||
ReadOptions ro; |
||||
ro.total_order_seek = true; |
||||
Arena arena; |
||||
for (MemTable* m : mems) { |
||||
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, |
||||
"[%s] Flushing memtable with next log file: %" PRIu64 "\n", |
||||
cfd_->GetName().c_str(), m->GetNextLogNumber()); |
||||
memtables.push_back(m->NewIterator(ro, &arena)); |
||||
} |
||||
{ |
||||
ScopedArenaIterator iter( |
||||
NewMergingIterator(&cfd_->internal_comparator(), &memtables[0], |
||||
static_cast<int>(memtables.size()), &arena)); |
||||
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, |
||||
"[%s] Level-0 flush table #%" PRIu64 ": started", |
||||
cfd_->GetName().c_str(), meta.fd.GetNumber()); |
||||
|
||||
s = BuildTable(dbname_, db_options_.env, *cfd_->ioptions(), env_options_, |
||||
cfd_->table_cache(), iter.get(), &meta, |
||||
cfd_->internal_comparator(), newest_snapshot_, |
||||
earliest_seqno_in_memtable, output_compression_, |
||||
cfd_->ioptions()->compression_opts, Env::IO_HIGH); |
||||
LogFlush(db_options_.info_log); |
||||
} |
||||
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, |
||||
"[%s] Level-0 flush table #%" PRIu64 ": %" PRIu64 " bytes %s", |
||||
cfd_->GetName().c_str(), meta.fd.GetNumber(), meta.fd.GetFileSize(), |
||||
s.ToString().c_str()); |
||||
if (!db_options_.disableDataSync && output_file_directory_ != nullptr) { |
||||
output_file_directory_->Fsync(); |
||||
} |
||||
db_mutex_->Lock(); |
||||
} |
||||
base->Unref(); |
||||
|
||||
// re-acquire the most current version
|
||||
base = cfd_->current(); |
||||
|
||||
// Note that if file_size is zero, the file has been deleted and
|
||||
// should not be added to the manifest.
|
||||
int level = 0; |
||||
if (s.ok() && meta.fd.GetFileSize() > 0) { |
||||
const Slice min_user_key = meta.smallest.user_key(); |
||||
const Slice max_user_key = meta.largest.user_key(); |
||||
// if we have more than 1 background thread, then we cannot
|
||||
// insert files directly into higher levels because some other
|
||||
// threads could be concurrently producing compacted files for
|
||||
// that key range.
|
||||
if (base != nullptr && db_options_.max_background_compactions <= 1 && |
||||
db_options_.max_background_flushes == 0 && |
||||
cfd_->ioptions()->compaction_style == kCompactionStyleLevel) { |
||||
level = base->storage_info()->PickLevelForMemTableOutput( |
||||
mutable_cf_options_, min_user_key, max_user_key); |
||||
// If level does not match path id, reset level back to 0
|
||||
uint32_t fdpath = LevelCompactionPicker::GetPathId( |
||||
*cfd_->ioptions(), mutable_cf_options_, level); |
||||
if (fdpath != 0) { |
||||
level = 0; |
||||
} |
||||
} |
||||
edit->AddFile(level, meta.fd.GetNumber(), meta.fd.GetPathId(), |
||||
meta.fd.GetFileSize(), meta.smallest, meta.largest, |
||||
meta.smallest_seqno, meta.largest_seqno); |
||||
} |
||||
|
||||
InternalStats::CompactionStats stats(1); |
||||
stats.micros = db_options_.env->NowMicros() - start_micros; |
||||
stats.bytes_written = meta.fd.GetFileSize(); |
||||
cfd_->internal_stats()->AddCompactionStats(level, stats); |
||||
cfd_->internal_stats()->AddCFStats(InternalStats::BYTES_FLUSHED, |
||||
meta.fd.GetFileSize()); |
||||
RecordTick(stats_, COMPACT_WRITE_BYTES, meta.fd.GetFileSize()); |
||||
return s; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,87 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
#pragma once |
||||
|
||||
#include <atomic> |
||||
#include <deque> |
||||
#include <limits> |
||||
#include <set> |
||||
#include <utility> |
||||
#include <vector> |
||||
#include <string> |
||||
|
||||
#include "db/dbformat.h" |
||||
#include "db/log_writer.h" |
||||
#include "db/snapshot.h" |
||||
#include "db/column_family.h" |
||||
#include "db/version_edit.h" |
||||
#include "db/memtable_list.h" |
||||
#include "port/port.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/memtablerep.h" |
||||
#include "rocksdb/transaction_log.h" |
||||
#include "util/autovector.h" |
||||
#include "util/instrumented_mutex.h" |
||||
#include "util/stop_watch.h" |
||||
#include "util/thread_local.h" |
||||
#include "util/scoped_arena_iterator.h" |
||||
#include "db/internal_stats.h" |
||||
#include "db/write_controller.h" |
||||
#include "db/flush_scheduler.h" |
||||
#include "db/write_thread.h" |
||||
#include "db/job_context.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class MemTable; |
||||
class TableCache; |
||||
class Version; |
||||
class VersionEdit; |
||||
class VersionSet; |
||||
class Arena; |
||||
|
||||
class FlushJob { |
||||
public: |
||||
// TODO(icanadi) make effort to reduce number of parameters here
|
||||
// IMPORTANT: mutable_cf_options needs to be alive while FlushJob is alive
|
||||
FlushJob(const std::string& dbname, ColumnFamilyData* cfd, |
||||
const DBOptions& db_options, |
||||
const MutableCFOptions& mutable_cf_options, |
||||
const EnvOptions& env_options, VersionSet* versions, |
||||
InstrumentedMutex* db_mutex, std::atomic<bool>* shutting_down, |
||||
SequenceNumber newest_snapshot, JobContext* job_context, |
||||
LogBuffer* log_buffer, Directory* db_directory, |
||||
Directory* output_file_directory, CompressionType output_compression, |
||||
Statistics* stats); |
||||
~FlushJob() {} |
||||
|
||||
Status Run(uint64_t* file_number = nullptr); |
||||
|
||||
private: |
||||
Status WriteLevel0Table(const autovector<MemTable*>& mems, VersionEdit* edit, |
||||
uint64_t* filenumber); |
||||
const std::string& dbname_; |
||||
ColumnFamilyData* cfd_; |
||||
const DBOptions& db_options_; |
||||
const MutableCFOptions& mutable_cf_options_; |
||||
const EnvOptions& env_options_; |
||||
VersionSet* versions_; |
||||
InstrumentedMutex* db_mutex_; |
||||
std::atomic<bool>* shutting_down_; |
||||
SequenceNumber newest_snapshot_; |
||||
JobContext* job_context_; |
||||
LogBuffer* log_buffer_; |
||||
Directory* db_directory_; |
||||
Directory* output_file_directory_; |
||||
CompressionType output_compression_; |
||||
Statistics* stats_; |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,124 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <map> |
||||
#include <string> |
||||
|
||||
#include "db/flush_job.h" |
||||
#include "db/column_family.h" |
||||
#include "db/version_set.h" |
||||
#include "db/writebuffer.h" |
||||
#include "rocksdb/cache.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
#include "table/mock_table.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
// TODO(icanadi) Mock out everything else:
|
||||
// 1. VersionSet
|
||||
// 2. Memtable
|
||||
class FlushJobTest { |
||||
public: |
||||
FlushJobTest() |
||||
: env_(Env::Default()), |
||||
dbname_(test::TmpDir() + "/flush_job_test"), |
||||
table_cache_(NewLRUCache(50000, 16, 8)), |
||||
write_buffer_(db_options_.db_write_buffer_size), |
||||
versions_(new VersionSet(dbname_, &db_options_, env_options_, |
||||
table_cache_.get(), &write_buffer_, |
||||
&write_controller_)), |
||||
shutting_down_(false), |
||||
mock_table_factory_(new mock::MockTableFactory()) { |
||||
ASSERT_OK(env_->CreateDirIfMissing(dbname_)); |
||||
db_options_.db_paths.emplace_back(dbname_, |
||||
std::numeric_limits<uint64_t>::max()); |
||||
// TODO(icanadi) Remove this once we mock out VersionSet
|
||||
NewDB(); |
||||
std::vector<ColumnFamilyDescriptor> column_families; |
||||
cf_options_.table_factory = mock_table_factory_; |
||||
column_families.emplace_back(kDefaultColumnFamilyName, cf_options_); |
||||
|
||||
ASSERT_OK(versions_->Recover(column_families, false)); |
||||
} |
||||
|
||||
void NewDB() { |
||||
VersionEdit new_db; |
||||
new_db.SetLogNumber(0); |
||||
new_db.SetNextFile(2); |
||||
new_db.SetLastSequence(0); |
||||
|
||||
const std::string manifest = DescriptorFileName(dbname_, 1); |
||||
unique_ptr<WritableFile> file; |
||||
Status s = env_->NewWritableFile( |
||||
manifest, &file, env_->OptimizeForManifestWrite(env_options_)); |
||||
ASSERT_OK(s); |
||||
{ |
||||
log::Writer log(std::move(file)); |
||||
std::string record; |
||||
new_db.EncodeTo(&record); |
||||
s = log.AddRecord(record); |
||||
} |
||||
ASSERT_OK(s); |
||||
// Make "CURRENT" file that points to the new manifest file.
|
||||
s = SetCurrentFile(env_, dbname_, 1, nullptr); |
||||
} |
||||
|
||||
Env* env_; |
||||
std::string dbname_; |
||||
EnvOptions env_options_; |
||||
std::shared_ptr<Cache> table_cache_; |
||||
WriteController write_controller_; |
||||
DBOptions db_options_; |
||||
WriteBuffer write_buffer_; |
||||
ColumnFamilyOptions cf_options_; |
||||
std::unique_ptr<VersionSet> versions_; |
||||
InstrumentedMutex mutex_; |
||||
std::atomic<bool> shutting_down_; |
||||
std::shared_ptr<mock::MockTableFactory> mock_table_factory_; |
||||
}; |
||||
|
||||
TEST(FlushJobTest, Empty) { |
||||
JobContext job_context; |
||||
auto cfd = versions_->GetColumnFamilySet()->GetDefault(); |
||||
FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(), |
||||
db_options_, *cfd->GetLatestMutableCFOptions(), |
||||
env_options_, versions_.get(), &mutex_, &shutting_down_, |
||||
SequenceNumber(), &job_context, nullptr, nullptr, nullptr, |
||||
kNoCompression, nullptr); |
||||
ASSERT_OK(flush_job.Run()); |
||||
job_context.Clean(); |
||||
} |
||||
|
||||
TEST(FlushJobTest, NonEmpty) { |
||||
JobContext job_context; |
||||
auto cfd = versions_->GetColumnFamilySet()->GetDefault(); |
||||
auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions()); |
||||
new_mem->Ref(); |
||||
std::map<std::string, std::string> inserted_keys; |
||||
for (int i = 1; i < 10000; ++i) { |
||||
std::string key(ToString(i)); |
||||
std::string value("value" + ToString(i)); |
||||
new_mem->Add(SequenceNumber(i), kTypeValue, key, value); |
||||
InternalKey internal_key(key, SequenceNumber(i), kTypeValue); |
||||
inserted_keys.insert({internal_key.Encode().ToString(), value}); |
||||
} |
||||
cfd->imm()->Add(new_mem); |
||||
|
||||
FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(), |
||||
db_options_, *cfd->GetLatestMutableCFOptions(), |
||||
env_options_, versions_.get(), &mutex_, &shutting_down_, |
||||
SequenceNumber(), &job_context, nullptr, nullptr, nullptr, |
||||
kNoCompression, nullptr); |
||||
mutex_.Lock(); |
||||
ASSERT_OK(flush_job.Run()); |
||||
mutex_.Unlock(); |
||||
mock_table_factory_->AssertSingleFile(inserted_keys); |
||||
job_context.Clean(); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } |
||||
@ -0,0 +1,63 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include "db/flush_scheduler.h" |
||||
|
||||
#include <cassert> |
||||
|
||||
#include "db/column_family.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
void FlushScheduler::ScheduleFlush(ColumnFamilyData* cfd) { |
||||
#ifndef NDEBUG |
||||
assert(column_families_set_.find(cfd) == column_families_set_.end()); |
||||
column_families_set_.insert(cfd); |
||||
#endif // NDEBUG
|
||||
cfd->Ref(); |
||||
column_families_.push_back(cfd); |
||||
} |
||||
|
||||
ColumnFamilyData* FlushScheduler::GetNextColumnFamily() { |
||||
ColumnFamilyData* cfd = nullptr; |
||||
while (column_families_.size() > 0) { |
||||
cfd = column_families_.front(); |
||||
column_families_.pop_front(); |
||||
if (cfd->IsDropped()) { |
||||
if (cfd->Unref()) { |
||||
delete cfd; |
||||
cfd = nullptr; |
||||
} |
||||
} else { |
||||
break; |
||||
} |
||||
} |
||||
#ifndef NDEBUG |
||||
if (cfd != nullptr) { |
||||
auto itr = column_families_set_.find(cfd); |
||||
assert(itr != column_families_set_.end()); |
||||
column_families_set_.erase(itr); |
||||
} |
||||
#endif // NDEBUG
|
||||
return cfd; |
||||
} |
||||
|
||||
bool FlushScheduler::Empty() { return column_families_.empty(); } |
||||
|
||||
void FlushScheduler::Clear() { |
||||
for (auto cfd : column_families_) { |
||||
#ifndef NDEBUG |
||||
auto itr = column_families_set_.find(cfd); |
||||
assert(itr != column_families_set_.end()); |
||||
column_families_set_.erase(itr); |
||||
#endif // NDEBUG
|
||||
if (cfd->Unref()) { |
||||
delete cfd; |
||||
} |
||||
} |
||||
column_families_.clear(); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,40 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#pragma once |
||||
|
||||
#include <stdint.h> |
||||
#include <deque> |
||||
#include <set> |
||||
#include <vector> |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class ColumnFamilyData; |
||||
|
||||
// This class is thread-compatible. It's should only be accessed from single
|
||||
// write thread (between BeginWrite() and EndWrite())
|
||||
class FlushScheduler { |
||||
public: |
||||
FlushScheduler() = default; |
||||
~FlushScheduler() = default; |
||||
|
||||
void ScheduleFlush(ColumnFamilyData* cfd); |
||||
// Returns Ref()-ed column family. Client needs to Unref()
|
||||
// REQUIRES: db mutex is held (exception is single-threaded recovery)
|
||||
ColumnFamilyData* GetNextColumnFamily(); |
||||
|
||||
bool Empty(); |
||||
|
||||
void Clear(); |
||||
|
||||
private: |
||||
std::deque<ColumnFamilyData*> column_families_; |
||||
#ifndef NDEBUG |
||||
std::set<ColumnFamilyData*> column_families_set_; |
||||
#endif // NDEBUG
|
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,103 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once |
||||
|
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
#include "db/column_family.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class MemTable; |
||||
|
||||
struct JobContext { |
||||
inline bool HaveSomethingToDelete() const { |
||||
return full_scan_candidate_files.size() || sst_delete_files.size() || |
||||
log_delete_files.size() || new_superversion != nullptr || |
||||
superversions_to_free.size() > 0 || memtables_to_free.size() > 0; |
||||
} |
||||
|
||||
// Structure to store information for candidate files to delete.
|
||||
struct CandidateFileInfo { |
||||
std::string file_name; |
||||
uint32_t path_id; |
||||
CandidateFileInfo(std::string name, uint32_t path) |
||||
: file_name(std::move(name)), path_id(path) {} |
||||
bool operator==(const CandidateFileInfo& other) const { |
||||
return file_name == other.file_name && path_id == other.path_id; |
||||
} |
||||
}; |
||||
|
||||
// a list of all files that we'll consider deleting
|
||||
// (every once in a while this is filled up with all files
|
||||
// in the DB directory)
|
||||
// (filled only if we're doing full scan)
|
||||
std::vector<CandidateFileInfo> full_scan_candidate_files; |
||||
|
||||
// the list of all live sst files that cannot be deleted
|
||||
std::vector<FileDescriptor> sst_live; |
||||
|
||||
// a list of sst files that we need to delete
|
||||
std::vector<FileMetaData*> sst_delete_files; |
||||
|
||||
// a list of log files that we need to delete
|
||||
std::vector<uint64_t> log_delete_files; |
||||
|
||||
// a list of memtables to be free
|
||||
autovector<MemTable*> memtables_to_free; |
||||
|
||||
autovector<SuperVersion*> superversions_to_free; |
||||
|
||||
SuperVersion* new_superversion; // if nullptr no new superversion
|
||||
|
||||
// the current manifest_file_number, log_number and prev_log_number
|
||||
// that corresponds to the set of files in 'live'.
|
||||
uint64_t manifest_file_number; |
||||
uint64_t pending_manifest_file_number; |
||||
uint64_t log_number; |
||||
uint64_t prev_log_number; |
||||
|
||||
uint64_t min_pending_output = 0; |
||||
|
||||
explicit JobContext(bool create_superversion = false) { |
||||
manifest_file_number = 0; |
||||
pending_manifest_file_number = 0; |
||||
log_number = 0; |
||||
prev_log_number = 0; |
||||
new_superversion = create_superversion ? new SuperVersion() : nullptr; |
||||
} |
||||
|
||||
void Clean() { |
||||
// free pending memtables
|
||||
for (auto m : memtables_to_free) { |
||||
delete m; |
||||
} |
||||
// free superversions
|
||||
for (auto s : superversions_to_free) { |
||||
delete s; |
||||
} |
||||
// if new_superversion was not used, it will be non-nullptr and needs
|
||||
// to be freed here
|
||||
delete new_superversion; |
||||
|
||||
memtables_to_free.clear(); |
||||
superversions_to_free.clear(); |
||||
new_superversion = nullptr; |
||||
} |
||||
|
||||
~JobContext() { |
||||
assert(memtables_to_free.size() == 0); |
||||
assert(superversions_to_free.size() == 0); |
||||
assert(new_superversion == nullptr); |
||||
} |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,401 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
#include "db/dbformat.h" |
||||
#include "db/db_impl.h" |
||||
#include "db/filename.h" |
||||
#include "db/version_set.h" |
||||
#include "db/write_batch_internal.h" |
||||
#include "rocksdb/cache.h" |
||||
#include "rocksdb/compaction_filter.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/filter_policy.h" |
||||
#include "rocksdb/perf_context.h" |
||||
#include "rocksdb/slice.h" |
||||
#include "rocksdb/slice_transform.h" |
||||
#include "rocksdb/table.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/table_properties.h" |
||||
#include "table/block_based_table_factory.h" |
||||
#include "table/plain_table_factory.h" |
||||
#include "util/hash.h" |
||||
#include "util/hash_linklist_rep.h" |
||||
#include "utilities/merge_operators.h" |
||||
#include "util/logging.h" |
||||
#include "util/mutexlock.h" |
||||
#include "util/rate_limiter.h" |
||||
#include "util/statistics.h" |
||||
#include "util/testharness.h" |
||||
#include "util/sync_point.h" |
||||
#include "util/testutil.h" |
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class EventListenerTest { |
||||
public: |
||||
EventListenerTest() { |
||||
dbname_ = test::TmpDir() + "/listener_test"; |
||||
ASSERT_OK(DestroyDB(dbname_, Options())); |
||||
db_ = nullptr; |
||||
Reopen(); |
||||
} |
||||
|
||||
~EventListenerTest() { |
||||
Close(); |
||||
Options options; |
||||
options.db_paths.emplace_back(dbname_, 0); |
||||
options.db_paths.emplace_back(dbname_ + "_2", 0); |
||||
options.db_paths.emplace_back(dbname_ + "_3", 0); |
||||
options.db_paths.emplace_back(dbname_ + "_4", 0); |
||||
ASSERT_OK(DestroyDB(dbname_, options)); |
||||
} |
||||
|
||||
void CreateColumnFamilies(const std::vector<std::string>& cfs, |
||||
const ColumnFamilyOptions* options = nullptr) { |
||||
ColumnFamilyOptions cf_opts; |
||||
cf_opts = ColumnFamilyOptions(Options()); |
||||
size_t cfi = handles_.size(); |
||||
handles_.resize(cfi + cfs.size()); |
||||
for (auto cf : cfs) { |
||||
ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++])); |
||||
} |
||||
} |
||||
|
||||
void Close() { |
||||
for (auto h : handles_) { |
||||
delete h; |
||||
} |
||||
handles_.clear(); |
||||
delete db_; |
||||
db_ = nullptr; |
||||
} |
||||
|
||||
void ReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
||||
const Options* options = nullptr) { |
||||
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options)); |
||||
} |
||||
|
||||
Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs, |
||||
const Options* options = nullptr) { |
||||
Close(); |
||||
Options opts = (options == nullptr) ? Options() : *options; |
||||
std::vector<const Options*> v_opts(cfs.size(), &opts); |
||||
return TryReopenWithColumnFamilies(cfs, v_opts); |
||||
} |
||||
|
||||
Status TryReopenWithColumnFamilies( |
||||
const std::vector<std::string>& cfs, |
||||
const std::vector<const Options*>& options) { |
||||
Close(); |
||||
ASSERT_EQ(cfs.size(), options.size()); |
||||
std::vector<ColumnFamilyDescriptor> column_families; |
||||
for (size_t i = 0; i < cfs.size(); ++i) { |
||||
column_families.push_back(ColumnFamilyDescriptor(cfs[i], *options[i])); |
||||
} |
||||
DBOptions db_opts = DBOptions(*options[0]); |
||||
return DB::Open(db_opts, dbname_, column_families, &handles_, &db_); |
||||
} |
||||
|
||||
Status TryReopen(Options* options = nullptr) { |
||||
Close(); |
||||
Options opts; |
||||
if (options != nullptr) { |
||||
opts = *options; |
||||
} else { |
||||
opts.create_if_missing = true; |
||||
} |
||||
|
||||
return DB::Open(opts, dbname_, &db_); |
||||
} |
||||
|
||||
void Reopen(Options* options = nullptr) { |
||||
ASSERT_OK(TryReopen(options)); |
||||
} |
||||
|
||||
void CreateAndReopenWithCF(const std::vector<std::string>& cfs, |
||||
const Options* options = nullptr) { |
||||
CreateColumnFamilies(cfs, options); |
||||
std::vector<std::string> cfs_plus_default = cfs; |
||||
cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName); |
||||
ReopenWithColumnFamilies(cfs_plus_default, options); |
||||
} |
||||
|
||||
DBImpl* dbfull() { |
||||
return reinterpret_cast<DBImpl*>(db_); |
||||
} |
||||
|
||||
Status Put(int cf, const Slice& k, const Slice& v, |
||||
WriteOptions wo = WriteOptions()) { |
||||
return db_->Put(wo, handles_[cf], k, v); |
||||
} |
||||
|
||||
Status Flush(int cf = 0) { |
||||
FlushOptions opt = FlushOptions(); |
||||
opt.wait = true; |
||||
if (cf == 0) { |
||||
return db_->Flush(opt); |
||||
} else { |
||||
return db_->Flush(opt, handles_[cf]); |
||||
} |
||||
} |
||||
|
||||
DB* db_; |
||||
std::string dbname_; |
||||
std::vector<ColumnFamilyHandle*> handles_; |
||||
}; |
||||
|
||||
class TestCompactionListener : public EventListener { |
||||
public: |
||||
void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) override { |
||||
compacted_dbs_.push_back(db); |
||||
} |
||||
|
||||
std::vector<DB*> compacted_dbs_; |
||||
}; |
||||
|
||||
TEST(EventListenerTest, OnSingleDBCompactionTest) { |
||||
const int kTestKeySize = 16; |
||||
const int kTestValueSize = 984; |
||||
const int kEntrySize = kTestKeySize + kTestValueSize; |
||||
const int kEntriesPerBuffer = 100; |
||||
const int kNumL0Files = 4; |
||||
|
||||
Options options; |
||||
options.create_if_missing = true; |
||||
options.write_buffer_size = kEntrySize * kEntriesPerBuffer; |
||||
options.compaction_style = kCompactionStyleLevel; |
||||
options.target_file_size_base = options.write_buffer_size; |
||||
options.max_bytes_for_level_base = options.target_file_size_base * 2; |
||||
options.max_bytes_for_level_multiplier = 2; |
||||
options.compression = kNoCompression; |
||||
options.enable_thread_tracking = true; |
||||
options.level0_file_num_compaction_trigger = kNumL0Files; |
||||
|
||||
TestCompactionListener* listener = new TestCompactionListener(); |
||||
options.listeners.emplace_back(listener); |
||||
std::vector<std::string> cf_names = { |
||||
"pikachu", "ilya", "muromec", "dobrynia", |
||||
"nikitich", "alyosha", "popovich"}; |
||||
CreateAndReopenWithCF(cf_names, &options); |
||||
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p'))); |
||||
ASSERT_OK(Put(2, "ilya", std::string(90000, 'i'))); |
||||
ASSERT_OK(Put(3, "muromec", std::string(90000, 'm'))); |
||||
ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd'))); |
||||
ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n'))); |
||||
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); |
||||
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); |
||||
for (size_t i = 1; i < 8; ++i) { |
||||
ASSERT_OK(Flush(static_cast<int>(i))); |
||||
const Slice kStart = "a"; |
||||
const Slice kEnd = "z"; |
||||
ASSERT_OK(dbfull()->CompactRange(handles_[i], &kStart, &kEnd)); |
||||
dbfull()->TEST_WaitForFlushMemTable(); |
||||
dbfull()->TEST_WaitForCompact(); |
||||
} |
||||
|
||||
ASSERT_EQ(listener->compacted_dbs_.size(), cf_names.size()); |
||||
for (size_t i = 0; i < cf_names.size(); ++i) { |
||||
ASSERT_EQ(listener->compacted_dbs_[i], db_); |
||||
} |
||||
} |
||||
|
||||
class TestFlushListener : public EventListener { |
||||
public: |
||||
void OnFlushCompleted( |
||||
DB* db, const std::string& name, |
||||
const std::string& file_path, |
||||
bool triggered_writes_slowdown, |
||||
bool triggered_writes_stop) override { |
||||
flushed_dbs_.push_back(db); |
||||
flushed_column_family_names_.push_back(name); |
||||
if (triggered_writes_slowdown) { |
||||
slowdown_count++; |
||||
} |
||||
if (triggered_writes_stop) { |
||||
stop_count++; |
||||
} |
||||
} |
||||
|
||||
std::vector<std::string> flushed_column_family_names_; |
||||
std::vector<DB*> flushed_dbs_; |
||||
int slowdown_count; |
||||
int stop_count; |
||||
}; |
||||
|
||||
TEST(EventListenerTest, OnSingleDBFlushTest) { |
||||
Options options; |
||||
options.write_buffer_size = 100000; |
||||
TestFlushListener* listener = new TestFlushListener(); |
||||
options.listeners.emplace_back(listener); |
||||
std::vector<std::string> cf_names = { |
||||
"pikachu", "ilya", "muromec", "dobrynia", |
||||
"nikitich", "alyosha", "popovich"}; |
||||
CreateAndReopenWithCF(cf_names, &options); |
||||
|
||||
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p'))); |
||||
ASSERT_OK(Put(2, "ilya", std::string(90000, 'i'))); |
||||
ASSERT_OK(Put(3, "muromec", std::string(90000, 'm'))); |
||||
ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd'))); |
||||
ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n'))); |
||||
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); |
||||
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); |
||||
for (size_t i = 1; i < 8; ++i) { |
||||
ASSERT_OK(Flush(static_cast<int>(i))); |
||||
dbfull()->TEST_WaitForFlushMemTable(); |
||||
ASSERT_EQ(listener->flushed_dbs_.size(), i); |
||||
ASSERT_EQ(listener->flushed_column_family_names_.size(), i); |
||||
} |
||||
|
||||
// make sure call-back functions are called in the right order
|
||||
for (size_t i = 0; i < cf_names.size(); ++i) { |
||||
ASSERT_EQ(listener->flushed_dbs_[i], db_); |
||||
ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]); |
||||
} |
||||
} |
||||
|
||||
TEST(EventListenerTest, MultiCF) { |
||||
Options options; |
||||
options.write_buffer_size = 100000; |
||||
TestFlushListener* listener = new TestFlushListener(); |
||||
options.listeners.emplace_back(listener); |
||||
std::vector<std::string> cf_names = { |
||||
"pikachu", "ilya", "muromec", "dobrynia", |
||||
"nikitich", "alyosha", "popovich"}; |
||||
CreateAndReopenWithCF(cf_names, &options); |
||||
|
||||
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p'))); |
||||
ASSERT_OK(Put(2, "ilya", std::string(90000, 'i'))); |
||||
ASSERT_OK(Put(3, "muromec", std::string(90000, 'm'))); |
||||
ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd'))); |
||||
ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n'))); |
||||
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); |
||||
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); |
||||
for (size_t i = 1; i < 8; ++i) { |
||||
ASSERT_OK(Flush(static_cast<int>(i))); |
||||
ASSERT_EQ(listener->flushed_dbs_.size(), i); |
||||
ASSERT_EQ(listener->flushed_column_family_names_.size(), i); |
||||
} |
||||
|
||||
// make sure call-back functions are called in the right order
|
||||
for (size_t i = 0; i < cf_names.size(); i++) { |
||||
ASSERT_EQ(listener->flushed_dbs_[i], db_); |
||||
ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]); |
||||
} |
||||
} |
||||
|
||||
TEST(EventListenerTest, MultiDBMultiListeners) { |
||||
std::vector<TestFlushListener*> listeners; |
||||
const int kNumDBs = 5; |
||||
const int kNumListeners = 10; |
||||
for (int i = 0; i < kNumListeners; ++i) { |
||||
listeners.emplace_back(new TestFlushListener()); |
||||
} |
||||
|
||||
std::vector<std::string> cf_names = { |
||||
"pikachu", "ilya", "muromec", "dobrynia", |
||||
"nikitich", "alyosha", "popovich"}; |
||||
|
||||
Options options; |
||||
options.create_if_missing = true; |
||||
for (int i = 0; i < kNumListeners; ++i) { |
||||
options.listeners.emplace_back(listeners[i]); |
||||
} |
||||
DBOptions db_opts(options); |
||||
ColumnFamilyOptions cf_opts(options); |
||||
|
||||
std::vector<DB*> dbs; |
||||
std::vector<std::vector<ColumnFamilyHandle *>> vec_handles; |
||||
|
||||
for (int d = 0; d < kNumDBs; ++d) { |
||||
ASSERT_OK(DestroyDB(dbname_ + ToString(d), options)); |
||||
DB* db; |
||||
std::vector<ColumnFamilyHandle*> handles; |
||||
ASSERT_OK(DB::Open(options, dbname_ + ToString(d), &db)); |
||||
for (size_t c = 0; c < cf_names.size(); ++c) { |
||||
ColumnFamilyHandle* handle; |
||||
db->CreateColumnFamily(cf_opts, cf_names[c], &handle); |
||||
handles.push_back(handle); |
||||
} |
||||
|
||||
vec_handles.push_back(std::move(handles)); |
||||
dbs.push_back(db); |
||||
} |
||||
|
||||
for (int d = 0; d < kNumDBs; ++d) { |
||||
for (size_t c = 0; c < cf_names.size(); ++c) { |
||||
ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c], |
||||
cf_names[c], cf_names[c])); |
||||
} |
||||
} |
||||
|
||||
for (size_t c = 0; c < cf_names.size(); ++c) { |
||||
for (int d = 0; d < kNumDBs; ++d) { |
||||
ASSERT_OK(dbs[d]->Flush(FlushOptions(), vec_handles[d][c])); |
||||
reinterpret_cast<DBImpl*>(dbs[d])->TEST_WaitForFlushMemTable(); |
||||
} |
||||
} |
||||
|
||||
for (auto* listener : listeners) { |
||||
int pos = 0; |
||||
for (size_t c = 0; c < cf_names.size(); ++c) { |
||||
for (int d = 0; d < kNumDBs; ++d) { |
||||
ASSERT_EQ(listener->flushed_dbs_[pos], dbs[d]); |
||||
ASSERT_EQ(listener->flushed_column_family_names_[pos], cf_names[c]); |
||||
pos++; |
||||
} |
||||
} |
||||
} |
||||
|
||||
for (auto handles : vec_handles) { |
||||
for (auto h : handles) { |
||||
delete h; |
||||
} |
||||
handles.clear(); |
||||
} |
||||
vec_handles.clear(); |
||||
|
||||
for (auto db : dbs) { |
||||
delete db; |
||||
} |
||||
} |
||||
|
||||
TEST(EventListenerTest, DisableBGCompaction) { |
||||
Options options; |
||||
TestFlushListener* listener = new TestFlushListener(); |
||||
const int kSlowdownTrigger = 5; |
||||
const int kStopTrigger = 10; |
||||
options.level0_slowdown_writes_trigger = kSlowdownTrigger; |
||||
options.level0_stop_writes_trigger = kStopTrigger; |
||||
options.listeners.emplace_back(listener); |
||||
// BG compaction is disabled. Number of L0 files will simply keeps
|
||||
// increasing in this test.
|
||||
options.compaction_style = kCompactionStyleNone; |
||||
options.compression = kNoCompression; |
||||
options.write_buffer_size = 100000; // Small write buffer
|
||||
|
||||
CreateAndReopenWithCF({"pikachu"}, &options); |
||||
WriteOptions wopts; |
||||
wopts.timeout_hint_us = 100000; |
||||
ColumnFamilyMetaData cf_meta; |
||||
db_->GetColumnFamilyMetaData(handles_[1], &cf_meta); |
||||
// keep writing until writes are forced to stop.
|
||||
for (int i = 0; static_cast<int>(cf_meta.file_count) < kStopTrigger; ++i) { |
||||
Put(1, ToString(i), std::string(100000, 'x'), wopts); |
||||
db_->GetColumnFamilyMetaData(handles_[1], &cf_meta); |
||||
} |
||||
ASSERT_GE(listener->slowdown_count, kStopTrigger - kSlowdownTrigger); |
||||
ASSERT_GE(listener->stop_count, 1); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
int main(int argc, char** argv) { |
||||
return rocksdb::test::RunAllTests(); |
||||
} |
||||
|
||||
@ -0,0 +1,52 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <assert.h> |
||||
|
||||
#include "db/memtable_allocator.h" |
||||
#include "db/writebuffer.h" |
||||
#include "util/arena.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
MemTableAllocator::MemTableAllocator(Arena* arena, WriteBuffer* write_buffer) |
||||
: arena_(arena), write_buffer_(write_buffer), bytes_allocated_(0) { |
||||
} |
||||
|
||||
MemTableAllocator::~MemTableAllocator() { |
||||
DoneAllocating(); |
||||
} |
||||
|
||||
char* MemTableAllocator::Allocate(size_t bytes) { |
||||
assert(write_buffer_ != nullptr); |
||||
bytes_allocated_ += bytes; |
||||
write_buffer_->ReserveMem(bytes); |
||||
return arena_->Allocate(bytes); |
||||
} |
||||
|
||||
char* MemTableAllocator::AllocateAligned(size_t bytes, size_t huge_page_size, |
||||
Logger* logger) { |
||||
assert(write_buffer_ != nullptr); |
||||
bytes_allocated_ += bytes; |
||||
write_buffer_->ReserveMem(bytes); |
||||
return arena_->AllocateAligned(bytes, huge_page_size, logger); |
||||
} |
||||
|
||||
void MemTableAllocator::DoneAllocating() { |
||||
if (write_buffer_ != nullptr) { |
||||
write_buffer_->FreeMem(bytes_allocated_); |
||||
write_buffer_ = nullptr; |
||||
} |
||||
} |
||||
|
||||
size_t MemTableAllocator::BlockSize() const { |
||||
return arena_->BlockSize(); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,47 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// This is used by the MemTable to allocate write buffer memory. It connects
|
||||
// to WriteBuffer so we can track and enforce overall write buffer limits.
|
||||
|
||||
#pragma once |
||||
#include "util/allocator.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class Arena; |
||||
class Logger; |
||||
class WriteBuffer; |
||||
|
||||
class MemTableAllocator : public Allocator { |
||||
public: |
||||
explicit MemTableAllocator(Arena* arena, WriteBuffer* write_buffer); |
||||
~MemTableAllocator(); |
||||
|
||||
// Allocator interface
|
||||
char* Allocate(size_t bytes) override; |
||||
char* AllocateAligned(size_t bytes, size_t huge_page_size = 0, |
||||
Logger* logger = nullptr) override; |
||||
size_t BlockSize() const override; |
||||
|
||||
// Call when we're finished allocating memory so we can free it from
|
||||
// the write buffer's limit.
|
||||
void DoneAllocating(); |
||||
|
||||
private: |
||||
Arena* arena_; |
||||
WriteBuffer* write_buffer_; |
||||
size_t bytes_allocated_; |
||||
|
||||
// No copying allowed
|
||||
MemTableAllocator(const MemTableAllocator&); |
||||
void operator=(const MemTableAllocator&); |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,696 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#define __STDC_FORMAT_MACROS |
||||
|
||||
#ifndef GFLAGS |
||||
#include <cstdio> |
||||
int main() { |
||||
fprintf(stderr, "Please install gflags to run rocksdb tools\n"); |
||||
return 1; |
||||
} |
||||
#else |
||||
|
||||
#include <gflags/gflags.h> |
||||
|
||||
#include <atomic> |
||||
#include <iostream> |
||||
#include <memory> |
||||
#include <thread> |
||||
#include <type_traits> |
||||
#include <vector> |
||||
|
||||
#include "db/dbformat.h" |
||||
#include "db/memtable.h" |
||||
#include "db/writebuffer.h" |
||||
#include "port/port.h" |
||||
#include "port/stack_trace.h" |
||||
#include "rocksdb/comparator.h" |
||||
#include "rocksdb/memtablerep.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/slice_transform.h" |
||||
#include "util/arena.h" |
||||
#include "util/mutexlock.h" |
||||
#include "util/stop_watch.h" |
||||
#include "util/testutil.h" |
||||
|
||||
using GFLAGS::ParseCommandLineFlags; |
||||
using GFLAGS::RegisterFlagValidator; |
||||
using GFLAGS::SetUsageMessage; |
||||
|
||||
DEFINE_string(benchmarks, "fillrandom", |
||||
"Comma-separated list of benchmarks to run. Options:\n" |
||||
"\tfillrandom -- write N random values\n" |
||||
"\tfillseq -- write N values in sequential order\n" |
||||
"\treadrandom -- read N values in random order\n" |
||||
"\treadseq -- scan the DB\n" |
||||
"\treadwrite -- 1 thread writes while N - 1 threads " |
||||
"do random\n" |
||||
"\t reads\n" |
||||
"\tseqreadwrite -- 1 thread writes while N - 1 threads " |
||||
"do scans\n"); |
||||
|
||||
DEFINE_string(memtablerep, "skiplist", |
||||
"Which implementation of memtablerep to use. See " |
||||
"include/memtablerep.h for\n" |
||||
" more details. Options:\n" |
||||
"\tskiplist -- backed by a skiplist\n" |
||||
"\tvector -- backed by an std::vector\n" |
||||
"\thashskiplist -- backed by a hash skip list\n" |
||||
"\thashlinklist -- backed by a hash linked list\n" |
||||
"\tcuckoo -- backed by a cuckoo hash table"); |
||||
|
||||
DEFINE_int64(bucket_count, 1000000, |
||||
"bucket_count parameter to pass into NewHashSkiplistRepFactory or " |
||||
"NewHashLinkListRepFactory"); |
||||
|
||||
DEFINE_int32( |
||||
hashskiplist_height, 4, |
||||
"skiplist_height parameter to pass into NewHashSkiplistRepFactory"); |
||||
|
||||
DEFINE_int32( |
||||
hashskiplist_branching_factor, 4, |
||||
"branching_factor parameter to pass into NewHashSkiplistRepFactory"); |
||||
|
||||
DEFINE_int32( |
||||
huge_page_tlb_size, 0, |
||||
"huge_page_tlb_size parameter to pass into NewHashLinkListRepFactory"); |
||||
|
||||
DEFINE_int32(bucket_entries_logging_threshold, 4096, |
||||
"bucket_entries_logging_threshold parameter to pass into " |
||||
"NewHashLinkListRepFactory"); |
||||
|
||||
DEFINE_bool(if_log_bucket_dist_when_flash, true, |
||||
"if_log_bucket_dist_when_flash parameter to pass into " |
||||
"NewHashLinkListRepFactory"); |
||||
|
||||
DEFINE_int32( |
||||
threshold_use_skiplist, 256, |
||||
"threshold_use_skiplist parameter to pass into NewHashLinkListRepFactory"); |
||||
|
||||
DEFINE_int64( |
||||
write_buffer_size, 256, |
||||
"write_buffer_size parameter to pass into NewHashCuckooRepFactory"); |
||||
|
||||
DEFINE_int64( |
||||
average_data_size, 64, |
||||
"average_data_size parameter to pass into NewHashCuckooRepFactory"); |
||||
|
||||
DEFINE_int64( |
||||
hash_function_count, 4, |
||||
"hash_function_count parameter to pass into NewHashCuckooRepFactory"); |
||||
|
||||
DEFINE_int32( |
||||
num_threads, 1, |
||||
"Number of concurrent threads to run. If the benchmark includes writes,\n" |
||||
"then at most one thread will be a writer"); |
||||
|
||||
DEFINE_int32(num_operations, 1000000, |
||||
"Number of operations to do for write and random read benchmarks"); |
||||
|
||||
DEFINE_int32(num_scans, 10, |
||||
"Number of times for each thread to scan the memtablerep for " |
||||
"sequential read " |
||||
"benchmarks"); |
||||
|
||||
DEFINE_int32(item_size, 100, "Number of bytes each item should be"); |
||||
|
||||
DEFINE_int32(prefix_length, 8, |
||||
"Prefix length to pass into NewFixedPrefixTransform"); |
||||
|
||||
/* VectorRep settings */ |
||||
DEFINE_int64(vectorrep_count, 0, |
||||
"Number of entries to reserve on VectorRep initialization"); |
||||
|
||||
DEFINE_int64(seed, 0, |
||||
"Seed base for random number generators. " |
||||
"When 0 it is deterministic."); |
||||
|
||||
static rocksdb::Env* FLAGS_env = rocksdb::Env::Default(); |
||||
|
||||
namespace rocksdb { |
||||
|
||||
namespace { |
||||
struct CallbackVerifyArgs { |
||||
bool found; |
||||
LookupKey* key; |
||||
MemTableRep* table; |
||||
InternalKeyComparator* comparator; |
||||
}; |
||||
} // namespace
|
||||
|
||||
// Helper for quickly generating random data.
|
||||
class RandomGenerator { |
||||
private: |
||||
std::string data_; |
||||
unsigned int pos_; |
||||
|
||||
public: |
||||
RandomGenerator() { |
||||
Random rnd(301); |
||||
auto size = (unsigned)std::max(1048576, FLAGS_item_size); |
||||
test::RandomString(&rnd, size, &data_); |
||||
pos_ = 0; |
||||
} |
||||
|
||||
Slice Generate(unsigned int len) { |
||||
assert(len <= data_.size()); |
||||
if (pos_ + len > data_.size()) { |
||||
pos_ = 0; |
||||
} |
||||
pos_ += len; |
||||
return Slice(data_.data() + pos_ - len, len); |
||||
} |
||||
}; |
||||
|
||||
enum WriteMode { SEQUENTIAL, RANDOM, UNIQUE_RANDOM }; |
||||
|
||||
class KeyGenerator { |
||||
public: |
||||
KeyGenerator(Random64* rand, WriteMode mode, uint64_t num) |
||||
: rand_(rand), mode_(mode), num_(num), next_(0) { |
||||
if (mode_ == UNIQUE_RANDOM) { |
||||
// NOTE: if memory consumption of this approach becomes a concern,
|
||||
// we can either break it into pieces and only random shuffle a section
|
||||
// each time. Alternatively, use a bit map implementation
|
||||
// (https://reviews.facebook.net/differential/diff/54627/)
|
||||
values_.resize(num_); |
||||
for (uint64_t i = 0; i < num_; ++i) { |
||||
values_[i] = i; |
||||
} |
||||
std::shuffle( |
||||
values_.begin(), values_.end(), |
||||
std::default_random_engine(static_cast<unsigned int>(FLAGS_seed))); |
||||
} |
||||
} |
||||
|
||||
uint64_t Next() { |
||||
switch (mode_) { |
||||
case SEQUENTIAL: |
||||
return next_++; |
||||
case RANDOM: |
||||
return rand_->Next() % num_; |
||||
case UNIQUE_RANDOM: |
||||
return values_[next_++]; |
||||
} |
||||
assert(false); |
||||
return std::numeric_limits<uint64_t>::max(); |
||||
} |
||||
|
||||
private: |
||||
Random64* rand_; |
||||
WriteMode mode_; |
||||
const uint64_t num_; |
||||
uint64_t next_; |
||||
std::vector<uint64_t> values_; |
||||
}; |
||||
|
||||
class BenchmarkThread { |
||||
public: |
||||
explicit BenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* bytes_written, uint64_t* bytes_read, |
||||
uint64_t* sequence, uint64_t num_ops, |
||||
uint64_t* read_hits) |
||||
: table_(table), |
||||
key_gen_(key_gen), |
||||
bytes_written_(bytes_written), |
||||
bytes_read_(bytes_read), |
||||
sequence_(sequence), |
||||
num_ops_(num_ops), |
||||
read_hits_(read_hits) {} |
||||
|
||||
virtual void operator()() = 0; |
||||
virtual ~BenchmarkThread() {} |
||||
|
||||
protected: |
||||
MemTableRep* table_; |
||||
KeyGenerator* key_gen_; |
||||
uint64_t* bytes_written_; |
||||
uint64_t* bytes_read_; |
||||
uint64_t* sequence_; |
||||
uint64_t num_ops_; |
||||
uint64_t* read_hits_; |
||||
RandomGenerator generator_; |
||||
}; |
||||
|
||||
class FillBenchmarkThread : public BenchmarkThread { |
||||
public: |
||||
FillBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* bytes_written, uint64_t* bytes_read, |
||||
uint64_t* sequence, uint64_t num_ops, uint64_t* read_hits) |
||||
: BenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, |
||||
num_ops, read_hits) {} |
||||
|
||||
void FillOne() { |
||||
char* buf = nullptr; |
||||
auto internal_key_size = 16; |
||||
auto encoded_len = |
||||
FLAGS_item_size + VarintLength(internal_key_size) + internal_key_size; |
||||
KeyHandle handle = table_->Allocate(encoded_len, &buf); |
||||
assert(buf != nullptr); |
||||
char* p = EncodeVarint32(buf, internal_key_size); |
||||
auto key = key_gen_->Next(); |
||||
EncodeFixed64(p, key); |
||||
p += 8; |
||||
EncodeFixed64(p, ++(*sequence_)); |
||||
p += 8; |
||||
Slice bytes = generator_.Generate(FLAGS_item_size); |
||||
memcpy(p, bytes.data(), FLAGS_item_size); |
||||
p += FLAGS_item_size; |
||||
assert(p == buf + encoded_len); |
||||
table_->Insert(handle); |
||||
*bytes_written_ += encoded_len; |
||||
} |
||||
|
||||
void operator()() override { |
||||
for (unsigned int i = 0; i < num_ops_; ++i) { |
||||
FillOne(); |
||||
} |
||||
} |
||||
}; |
||||
|
||||
class ConcurrentFillBenchmarkThread : public FillBenchmarkThread { |
||||
public: |
||||
ConcurrentFillBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* bytes_written, uint64_t* bytes_read, |
||||
uint64_t* sequence, uint64_t num_ops, |
||||
uint64_t* read_hits, |
||||
std::atomic_int* threads_done) |
||||
: FillBenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, |
||||
num_ops, read_hits) { |
||||
threads_done_ = threads_done; |
||||
} |
||||
|
||||
void operator()() override { |
||||
// # of read threads will be total threads - write threads (always 1). Loop
|
||||
// while all reads complete.
|
||||
while ((*threads_done_).load() < (FLAGS_num_threads - 1)) { |
||||
FillOne(); |
||||
} |
||||
} |
||||
|
||||
private: |
||||
std::atomic_int* threads_done_; |
||||
}; |
||||
|
||||
class ReadBenchmarkThread : public BenchmarkThread { |
||||
public: |
||||
ReadBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* bytes_written, uint64_t* bytes_read, |
||||
uint64_t* sequence, uint64_t num_ops, uint64_t* read_hits) |
||||
: BenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, |
||||
num_ops, read_hits) {} |
||||
|
||||
static bool callback(void* arg, const char* entry) { |
||||
CallbackVerifyArgs* callback_args = static_cast<CallbackVerifyArgs*>(arg); |
||||
assert(callback_args != nullptr); |
||||
uint32_t key_length; |
||||
const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); |
||||
if ((callback_args->comparator)->user_comparator()->Compare( |
||||
Slice(key_ptr, key_length - 8), callback_args->key->user_key()) == |
||||
0) { |
||||
callback_args->found = true; |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
void ReadOne() { |
||||
std::string user_key; |
||||
auto key = key_gen_->Next(); |
||||
PutFixed64(&user_key, key); |
||||
LookupKey lookup_key(user_key, *sequence_); |
||||
InternalKeyComparator internal_key_comp(BytewiseComparator()); |
||||
CallbackVerifyArgs verify_args; |
||||
verify_args.found = false; |
||||
verify_args.key = &lookup_key; |
||||
verify_args.table = table_; |
||||
verify_args.comparator = &internal_key_comp; |
||||
table_->Get(lookup_key, &verify_args, callback); |
||||
if (verify_args.found) { |
||||
*bytes_read_ += VarintLength(16) + 16 + FLAGS_item_size; |
||||
++*read_hits_; |
||||
} |
||||
} |
||||
void operator()() override { |
||||
for (unsigned int i = 0; i < num_ops_; ++i) { |
||||
ReadOne(); |
||||
} |
||||
} |
||||
}; |
||||
|
||||
class SeqReadBenchmarkThread : public BenchmarkThread { |
||||
public: |
||||
SeqReadBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* bytes_written, uint64_t* bytes_read, |
||||
uint64_t* sequence, uint64_t num_ops, |
||||
uint64_t* read_hits) |
||||
: BenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, |
||||
num_ops, read_hits) {} |
||||
|
||||
void ReadOneSeq() { |
||||
std::unique_ptr<MemTableRep::Iterator> iter(table_->GetIterator()); |
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { |
||||
// pretend to read the value
|
||||
*bytes_read_ += VarintLength(16) + 16 + FLAGS_item_size; |
||||
} |
||||
++*read_hits_; |
||||
} |
||||
|
||||
void operator()() override { |
||||
for (unsigned int i = 0; i < num_ops_; ++i) { |
||||
{ ReadOneSeq(); } |
||||
} |
||||
} |
||||
}; |
||||
|
||||
class ConcurrentReadBenchmarkThread : public ReadBenchmarkThread { |
||||
public: |
||||
ConcurrentReadBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* bytes_written, uint64_t* bytes_read, |
||||
uint64_t* sequence, uint64_t num_ops, |
||||
uint64_t* read_hits, |
||||
std::atomic_int* threads_done) |
||||
: ReadBenchmarkThread(table, key_gen, bytes_written, bytes_read, sequence, |
||||
num_ops, read_hits) { |
||||
threads_done_ = threads_done; |
||||
} |
||||
|
||||
void operator()() override { |
||||
for (unsigned int i = 0; i < num_ops_; ++i) { |
||||
ReadOne(); |
||||
} |
||||
++*threads_done_; |
||||
} |
||||
|
||||
private: |
||||
std::atomic_int* threads_done_; |
||||
}; |
||||
|
||||
class SeqConcurrentReadBenchmarkThread : public SeqReadBenchmarkThread { |
||||
public: |
||||
SeqConcurrentReadBenchmarkThread(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* bytes_written, |
||||
uint64_t* bytes_read, uint64_t* sequence, |
||||
uint64_t num_ops, uint64_t* read_hits, |
||||
std::atomic_int* threads_done) |
||||
: SeqReadBenchmarkThread(table, key_gen, bytes_written, bytes_read, |
||||
sequence, num_ops, read_hits) { |
||||
threads_done_ = threads_done; |
||||
} |
||||
|
||||
void operator()() override { |
||||
for (unsigned int i = 0; i < num_ops_; ++i) { |
||||
ReadOneSeq(); |
||||
} |
||||
++*threads_done_; |
||||
} |
||||
|
||||
private: |
||||
std::atomic_int* threads_done_; |
||||
}; |
||||
|
||||
class Benchmark { |
||||
public: |
||||
explicit Benchmark(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* sequence, uint32_t num_threads) |
||||
: table_(table), |
||||
key_gen_(key_gen), |
||||
sequence_(sequence), |
||||
num_threads_(num_threads) {} |
||||
|
||||
virtual ~Benchmark() {} |
||||
virtual void Run() { |
||||
std::cout << "Number of threads: " << num_threads_ << std::endl; |
||||
std::vector<std::thread> threads; |
||||
uint64_t bytes_written = 0; |
||||
uint64_t bytes_read = 0; |
||||
uint64_t read_hits = 0; |
||||
StopWatchNano timer(Env::Default(), true); |
||||
RunThreads(&threads, &bytes_written, &bytes_read, true, &read_hits); |
||||
auto elapsed_time = static_cast<double>(timer.ElapsedNanos() / 1000); |
||||
std::cout << "Elapsed time: " << static_cast<int>(elapsed_time) << " us" |
||||
<< std::endl; |
||||
|
||||
if (bytes_written > 0) { |
||||
auto MiB_written = static_cast<double>(bytes_written) / (1 << 20); |
||||
auto write_throughput = MiB_written / (elapsed_time / 1000000); |
||||
std::cout << "Total bytes written: " << MiB_written << " MiB" |
||||
<< std::endl; |
||||
std::cout << "Write throughput: " << write_throughput << " MiB/s" |
||||
<< std::endl; |
||||
auto us_per_op = elapsed_time / num_write_ops_per_thread_; |
||||
std::cout << "write us/op: " << us_per_op << std::endl; |
||||
} |
||||
if (bytes_read > 0) { |
||||
auto MiB_read = static_cast<double>(bytes_read) / (1 << 20); |
||||
auto read_throughput = MiB_read / (elapsed_time / 1000000); |
||||
std::cout << "Total bytes read: " << MiB_read << " MiB" << std::endl; |
||||
std::cout << "Read throughput: " << read_throughput << " MiB/s" |
||||
<< std::endl; |
||||
auto us_per_op = elapsed_time / num_read_ops_per_thread_; |
||||
std::cout << "read us/op: " << us_per_op << std::endl; |
||||
} |
||||
} |
||||
|
||||
virtual void RunThreads(std::vector<std::thread>* threads, |
||||
uint64_t* bytes_written, uint64_t* bytes_read, |
||||
bool write, uint64_t* read_hits) = 0; |
||||
|
||||
protected: |
||||
MemTableRep* table_; |
||||
KeyGenerator* key_gen_; |
||||
uint64_t* sequence_; |
||||
uint64_t num_write_ops_per_thread_; |
||||
uint64_t num_read_ops_per_thread_; |
||||
const uint32_t num_threads_; |
||||
}; |
||||
|
||||
class FillBenchmark : public Benchmark { |
||||
public: |
||||
explicit FillBenchmark(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* sequence) |
||||
: Benchmark(table, key_gen, sequence, 1) { |
||||
num_write_ops_per_thread_ = FLAGS_num_operations; |
||||
} |
||||
|
||||
void RunThreads(std::vector<std::thread>* threads, uint64_t* bytes_written, |
||||
uint64_t* bytes_read, bool write, |
||||
uint64_t* read_hits) override { |
||||
FillBenchmarkThread(table_, key_gen_, bytes_written, bytes_read, sequence_, |
||||
num_write_ops_per_thread_, read_hits)(); |
||||
} |
||||
}; |
||||
|
||||
class ReadBenchmark : public Benchmark { |
||||
public: |
||||
explicit ReadBenchmark(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* sequence) |
||||
: Benchmark(table, key_gen, sequence, FLAGS_num_threads) { |
||||
num_read_ops_per_thread_ = FLAGS_num_operations / FLAGS_num_threads; |
||||
} |
||||
|
||||
void RunThreads(std::vector<std::thread>* threads, uint64_t* bytes_written, |
||||
uint64_t* bytes_read, bool write, |
||||
uint64_t* read_hits) override { |
||||
for (int i = 0; i < FLAGS_num_threads; ++i) { |
||||
threads->emplace_back( |
||||
ReadBenchmarkThread(table_, key_gen_, bytes_written, bytes_read, |
||||
sequence_, num_read_ops_per_thread_, read_hits)); |
||||
} |
||||
for (auto& thread : *threads) { |
||||
thread.join(); |
||||
} |
||||
std::cout << "read hit%: " |
||||
<< (static_cast<double>(*read_hits) / FLAGS_num_operations) * 100 |
||||
<< std::endl; |
||||
} |
||||
}; |
||||
|
||||
class SeqReadBenchmark : public Benchmark { |
||||
public: |
||||
explicit SeqReadBenchmark(MemTableRep* table, uint64_t* sequence) |
||||
: Benchmark(table, nullptr, sequence, FLAGS_num_threads) { |
||||
num_read_ops_per_thread_ = FLAGS_num_scans; |
||||
} |
||||
|
||||
void RunThreads(std::vector<std::thread>* threads, uint64_t* bytes_written, |
||||
uint64_t* bytes_read, bool write, |
||||
uint64_t* read_hits) override { |
||||
for (int i = 0; i < FLAGS_num_threads; ++i) { |
||||
threads->emplace_back(SeqReadBenchmarkThread( |
||||
table_, key_gen_, bytes_written, bytes_read, sequence_, |
||||
num_read_ops_per_thread_, read_hits)); |
||||
} |
||||
for (auto& thread : *threads) { |
||||
thread.join(); |
||||
} |
||||
} |
||||
}; |
||||
|
||||
template <class ReadThreadType> |
||||
class ReadWriteBenchmark : public Benchmark { |
||||
public: |
||||
explicit ReadWriteBenchmark(MemTableRep* table, KeyGenerator* key_gen, |
||||
uint64_t* sequence) |
||||
: Benchmark(table, key_gen, sequence, FLAGS_num_threads) { |
||||
num_read_ops_per_thread_ = |
||||
FLAGS_num_threads <= 1 |
||||
? 0 |
||||
: (FLAGS_num_operations / (FLAGS_num_threads - 1)); |
||||
num_write_ops_per_thread_ = FLAGS_num_operations; |
||||
} |
||||
|
||||
void RunThreads(std::vector<std::thread>* threads, uint64_t* bytes_written, |
||||
uint64_t* bytes_read, bool write, |
||||
uint64_t* read_hits) override { |
||||
std::atomic_int threads_done; |
||||
threads_done.store(0); |
||||
threads->emplace_back(ConcurrentFillBenchmarkThread( |
||||
table_, key_gen_, bytes_written, bytes_read, sequence_, |
||||
num_write_ops_per_thread_, read_hits, &threads_done)); |
||||
for (int i = 1; i < FLAGS_num_threads; ++i) { |
||||
threads->emplace_back( |
||||
ReadThreadType(table_, key_gen_, bytes_written, bytes_read, sequence_, |
||||
num_read_ops_per_thread_, read_hits, &threads_done)); |
||||
} |
||||
for (auto& thread : *threads) { |
||||
thread.join(); |
||||
} |
||||
} |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
void PrintWarnings() { |
||||
#if defined(__GNUC__) && !defined(__OPTIMIZE__) |
||||
fprintf(stdout, |
||||
"WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"); |
||||
#endif |
||||
#ifndef NDEBUG |
||||
fprintf(stdout, |
||||
"WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); |
||||
#endif |
||||
} |
||||
|
||||
int main(int argc, char** argv) { |
||||
rocksdb::port::InstallStackTraceHandler(); |
||||
SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + |
||||
" [OPTIONS]..."); |
||||
ParseCommandLineFlags(&argc, &argv, true); |
||||
|
||||
PrintWarnings(); |
||||
|
||||
rocksdb::Options options; |
||||
|
||||
std::unique_ptr<rocksdb::MemTableRepFactory> factory; |
||||
if (FLAGS_memtablerep == "skiplist") { |
||||
factory.reset(new rocksdb::SkipListFactory); |
||||
} else if (FLAGS_memtablerep == "vector") { |
||||
factory.reset(new rocksdb::VectorRepFactory); |
||||
} else if (FLAGS_memtablerep == "hashskiplist") { |
||||
factory.reset(rocksdb::NewHashSkipListRepFactory( |
||||
FLAGS_bucket_count, FLAGS_hashskiplist_height, |
||||
FLAGS_hashskiplist_branching_factor)); |
||||
options.prefix_extractor.reset( |
||||
rocksdb::NewFixedPrefixTransform(FLAGS_prefix_length)); |
||||
} else if (FLAGS_memtablerep == "hashlinklist") { |
||||
factory.reset(rocksdb::NewHashLinkListRepFactory( |
||||
FLAGS_bucket_count, FLAGS_huge_page_tlb_size, |
||||
FLAGS_bucket_entries_logging_threshold, |
||||
FLAGS_if_log_bucket_dist_when_flash, FLAGS_threshold_use_skiplist)); |
||||
options.prefix_extractor.reset( |
||||
rocksdb::NewFixedPrefixTransform(FLAGS_prefix_length)); |
||||
} else if (FLAGS_memtablerep == "cuckoo") { |
||||
factory.reset(rocksdb::NewHashCuckooRepFactory( |
||||
FLAGS_write_buffer_size, FLAGS_average_data_size, |
||||
static_cast<uint32_t>(FLAGS_hash_function_count))); |
||||
options.prefix_extractor.reset( |
||||
rocksdb::NewFixedPrefixTransform(FLAGS_prefix_length)); |
||||
} else { |
||||
fprintf(stdout, "Unknown memtablerep: %s\n", FLAGS_memtablerep.c_str()); |
||||
exit(1); |
||||
} |
||||
|
||||
rocksdb::InternalKeyComparator internal_key_comp( |
||||
rocksdb::BytewiseComparator()); |
||||
rocksdb::MemTable::KeyComparator key_comp(internal_key_comp); |
||||
rocksdb::Arena arena; |
||||
rocksdb::WriteBuffer wb(FLAGS_write_buffer_size); |
||||
rocksdb::MemTableAllocator memtable_allocator(&arena, &wb); |
||||
uint64_t sequence; |
||||
auto createMemtableRep = [&] { |
||||
sequence = 0; |
||||
return factory->CreateMemTableRep(key_comp, &memtable_allocator, |
||||
options.prefix_extractor.get(), |
||||
options.info_log.get()); |
||||
}; |
||||
std::unique_ptr<rocksdb::MemTableRep> memtablerep; |
||||
rocksdb::Random64 rng(FLAGS_seed); |
||||
const char* benchmarks = FLAGS_benchmarks.c_str(); |
||||
while (benchmarks != nullptr) { |
||||
std::unique_ptr<rocksdb::KeyGenerator> key_gen; |
||||
const char* sep = strchr(benchmarks, ','); |
||||
rocksdb::Slice name; |
||||
if (sep == nullptr) { |
||||
name = benchmarks; |
||||
benchmarks = nullptr; |
||||
} else { |
||||
name = rocksdb::Slice(benchmarks, sep - benchmarks); |
||||
benchmarks = sep + 1; |
||||
} |
||||
std::unique_ptr<rocksdb::Benchmark> benchmark; |
||||
if (name == rocksdb::Slice("fillseq")) { |
||||
memtablerep.reset(createMemtableRep()); |
||||
key_gen.reset(new rocksdb::KeyGenerator(&rng, rocksdb::SEQUENTIAL, |
||||
FLAGS_num_operations)); |
||||
benchmark.reset(new rocksdb::FillBenchmark(memtablerep.get(), |
||||
key_gen.get(), &sequence)); |
||||
} else if (name == rocksdb::Slice("fillrandom")) { |
||||
memtablerep.reset(createMemtableRep()); |
||||
key_gen.reset(new rocksdb::KeyGenerator(&rng, rocksdb::UNIQUE_RANDOM, |
||||
FLAGS_num_operations)); |
||||
benchmark.reset(new rocksdb::FillBenchmark(memtablerep.get(), |
||||
key_gen.get(), &sequence)); |
||||
} else if (name == rocksdb::Slice("readrandom")) { |
||||
key_gen.reset(new rocksdb::KeyGenerator(&rng, rocksdb::RANDOM, |
||||
FLAGS_num_operations)); |
||||
benchmark.reset(new rocksdb::ReadBenchmark(memtablerep.get(), |
||||
key_gen.get(), &sequence)); |
||||
} else if (name == rocksdb::Slice("readseq")) { |
||||
key_gen.reset(new rocksdb::KeyGenerator(&rng, rocksdb::SEQUENTIAL, |
||||
FLAGS_num_operations)); |
||||
benchmark.reset( |
||||
new rocksdb::SeqReadBenchmark(memtablerep.get(), &sequence)); |
||||
} else if (name == rocksdb::Slice("readwrite")) { |
||||
memtablerep.reset(createMemtableRep()); |
||||
key_gen.reset(new rocksdb::KeyGenerator(&rng, rocksdb::RANDOM, |
||||
FLAGS_num_operations)); |
||||
benchmark.reset(new rocksdb::ReadWriteBenchmark< |
||||
rocksdb::ConcurrentReadBenchmarkThread>(memtablerep.get(), |
||||
key_gen.get(), &sequence)); |
||||
} else if (name == rocksdb::Slice("seqreadwrite")) { |
||||
memtablerep.reset(createMemtableRep()); |
||||
key_gen.reset(new rocksdb::KeyGenerator(&rng, rocksdb::RANDOM, |
||||
FLAGS_num_operations)); |
||||
benchmark.reset(new rocksdb::ReadWriteBenchmark< |
||||
rocksdb::SeqConcurrentReadBenchmarkThread>(memtablerep.get(), |
||||
key_gen.get(), &sequence)); |
||||
} else { |
||||
std::cout << "WARNING: skipping unknown benchmark '" << name.ToString() |
||||
<< std::endl; |
||||
continue; |
||||
} |
||||
std::cout << "Running " << name.ToString() << std::endl; |
||||
benchmark->Run(); |
||||
} |
||||
|
||||
return 0; |
||||
} |
||||
|
||||
#endif // GFLAGS
|
||||
@ -1,810 +0,0 @@ |
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
#include <algorithm> |
||||
#include <set> |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/filter_policy.h" |
||||
#include "db/db_impl.h" |
||||
#include "db/filename.h" |
||||
#include "db/version_set.h" |
||||
#include "db/write_batch_internal.h" |
||||
#include "rocksdb/statistics.h" |
||||
#include "rocksdb/cache.h" |
||||
#include "rocksdb/compaction_filter.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/table.h" |
||||
#include "rocksdb/table_properties.h" |
||||
#include "table/table_builder.h" |
||||
#include "util/hash.h" |
||||
#include "util/logging.h" |
||||
#include "util/mutexlock.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
#include "utilities/merge_operators.h" |
||||
|
||||
using std::unique_ptr; |
||||
|
||||
// IS THIS FILE STILL NEEDED?
|
||||
namespace rocksdb { |
||||
|
||||
// SimpleTable is a simple table format for UNIT TEST ONLY. It is not built
|
||||
// as production quality.
|
||||
// SimpleTable requires the input key size to be fixed 16 bytes, value cannot
|
||||
// be longer than 150000 bytes and stored data on disk in this format:
|
||||
// +--------------------------------------------+ <= key1 offset
|
||||
// | key1 | value_size (4 bytes) | |
|
||||
// +----------------------------------------+ |
|
||||
// | value1 |
|
||||
// | |
|
||||
// +----------------------------------------+---+ <= key2 offset
|
||||
// | key2 | value_size (4 bytes) | |
|
||||
// +----------------------------------------+ |
|
||||
// | value2 |
|
||||
// | |
|
||||
// | ...... |
|
||||
// +-----------------+--------------------------+ <= index_block_offset
|
||||
// | key1 | key1 offset (8 bytes) |
|
||||
// +-----------------+--------------------------+
|
||||
// | key2 | key2 offset (8 bytes) |
|
||||
// +-----------------+--------------------------+
|
||||
// | key3 | key3 offset (8 bytes) |
|
||||
// +-----------------+--------------------------+
|
||||
// | ...... |
|
||||
// +-----------------+------------+-------------+
|
||||
// | index_block_offset (8 bytes) |
|
||||
// +------------------------------+
|
||||
|
||||
// SimpleTable is a simple table format for UNIT TEST ONLY. It is not built
|
||||
// as production quality.
|
||||
class SimpleTableReader: public TableReader { |
||||
public: |
||||
// Attempt to open the table that is stored in bytes [0..file_size)
|
||||
// of "file", and read the metadata entries necessary to allow
|
||||
// retrieving data from the table.
|
||||
//
|
||||
// If successful, returns ok and sets "*table" to the newly opened
|
||||
// table. The client should delete "*table" when no longer needed.
|
||||
// If there was an error while initializing the table, sets "*table"
|
||||
// to nullptr and returns a non-ok status. Does not take ownership of
|
||||
// "*source", but the client must ensure that "source" remains live
|
||||
// for the duration of the returned table's lifetime.
|
||||
//
|
||||
// *file must remain live while this Table is in use.
|
||||
static Status Open(const Options& options, const EnvOptions& soptions, |
||||
unique_ptr<RandomAccessFile> && file, uint64_t file_size, |
||||
unique_ptr<TableReader>* table_reader); |
||||
|
||||
Iterator* NewIterator(const ReadOptions&, Arena* arena) override; |
||||
|
||||
Status Get(const ReadOptions&, const Slice& key, void* arg, |
||||
bool (*handle_result)(void* arg, const ParsedInternalKey& k, |
||||
const Slice& v), |
||||
void (*mark_key_may_exist)(void*) = nullptr) override; |
||||
|
||||
uint64_t ApproximateOffsetOf(const Slice& key) override; |
||||
|
||||
virtual size_t ApproximateMemoryUsage() const override { return 0; } |
||||
|
||||
void SetupForCompaction() override; |
||||
|
||||
std::shared_ptr<const TableProperties> GetTableProperties() const override; |
||||
|
||||
~SimpleTableReader(); |
||||
|
||||
private: |
||||
struct Rep; |
||||
Rep* rep_; |
||||
|
||||
explicit SimpleTableReader(Rep* rep) { |
||||
rep_ = rep; |
||||
} |
||||
friend class TableCache; |
||||
friend class SimpleTableIterator; |
||||
|
||||
Status GetOffset(const Slice& target, uint64_t* offset); |
||||
|
||||
// No copying allowed
|
||||
explicit SimpleTableReader(const TableReader&) = delete; |
||||
void operator=(const TableReader&) = delete; |
||||
}; |
||||
|
||||
// Iterator to iterate SimpleTable
|
||||
class SimpleTableIterator: public Iterator { |
||||
public: |
||||
explicit SimpleTableIterator(SimpleTableReader* table); |
||||
~SimpleTableIterator(); |
||||
|
||||
bool Valid() const; |
||||
|
||||
void SeekToFirst(); |
||||
|
||||
void SeekToLast(); |
||||
|
||||
void Seek(const Slice& target); |
||||
|
||||
void Next(); |
||||
|
||||
void Prev(); |
||||
|
||||
Slice key() const; |
||||
|
||||
Slice value() const; |
||||
|
||||
Status status() const; |
||||
|
||||
private: |
||||
SimpleTableReader* table_; |
||||
uint64_t offset_; |
||||
uint64_t next_offset_; |
||||
Slice key_; |
||||
Slice value_; |
||||
char tmp_str_[4]; |
||||
char* key_str_; |
||||
char* value_str_; |
||||
int value_str_len_; |
||||
Status status_; |
||||
// No copying allowed
|
||||
SimpleTableIterator(const SimpleTableIterator&) = delete; |
||||
void operator=(const Iterator&) = delete; |
||||
}; |
||||
|
||||
struct SimpleTableReader::Rep { |
||||
~Rep() { |
||||
} |
||||
Rep(const EnvOptions& storage_options, uint64_t index_start_offset, |
||||
int num_entries) : |
||||
soptions(storage_options), index_start_offset(index_start_offset), |
||||
num_entries(num_entries) { |
||||
} |
||||
|
||||
Options options; |
||||
const EnvOptions& soptions; |
||||
Status status; |
||||
unique_ptr<RandomAccessFile> file; |
||||
uint64_t index_start_offset; |
||||
int num_entries; |
||||
std::shared_ptr<TableProperties> table_properties; |
||||
|
||||
const static int user_key_size = 16; |
||||
const static int offset_length = 8; |
||||
const static int key_footer_len = 8; |
||||
|
||||
static int GetInternalKeyLength() { |
||||
return user_key_size + key_footer_len; |
||||
} |
||||
}; |
||||
|
||||
SimpleTableReader::~SimpleTableReader() { |
||||
delete rep_; |
||||
} |
||||
|
||||
Status SimpleTableReader::Open(const Options& options, |
||||
const EnvOptions& soptions, |
||||
unique_ptr<RandomAccessFile> && file, |
||||
uint64_t size, |
||||
unique_ptr<TableReader>* table_reader) { |
||||
char footer_space[Rep::offset_length]; |
||||
Slice footer_input; |
||||
Status s = file->Read(size - Rep::offset_length, Rep::offset_length, |
||||
&footer_input, footer_space); |
||||
if (s.ok()) { |
||||
uint64_t index_start_offset = DecodeFixed64(footer_space); |
||||
|
||||
int num_entries = (size - Rep::offset_length - index_start_offset) |
||||
/ (Rep::GetInternalKeyLength() + Rep::offset_length); |
||||
SimpleTableReader::Rep* rep = new SimpleTableReader::Rep(soptions, |
||||
index_start_offset, |
||||
num_entries); |
||||
|
||||
rep->file = std::move(file); |
||||
rep->options = options; |
||||
table_reader->reset(new SimpleTableReader(rep)); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
void SimpleTableReader::SetupForCompaction() { |
||||
} |
||||
|
||||
std::shared_ptr<const TableProperties> SimpleTableReader::GetTableProperties() |
||||
const { |
||||
return rep_->table_properties; |
||||
} |
||||
|
||||
Iterator* SimpleTableReader::NewIterator(const ReadOptions& options, |
||||
Arena* arena) { |
||||
if (arena == nullptr) { |
||||
return new SimpleTableIterator(this); |
||||
} else { |
||||
auto mem = arena->AllocateAligned(sizeof(SimpleTableIterator)); |
||||
return new (mem) SimpleTableIterator(this); |
||||
} |
||||
} |
||||
|
||||
Status SimpleTableReader::GetOffset(const Slice& target, uint64_t* offset) { |
||||
uint32_t left = 0; |
||||
uint32_t right = rep_->num_entries - 1; |
||||
char key_chars[Rep::GetInternalKeyLength()]; |
||||
Slice tmp_slice; |
||||
|
||||
uint32_t target_offset = 0; |
||||
while (left <= right) { |
||||
uint32_t mid = (left + right + 1) / 2; |
||||
|
||||
uint64_t offset_to_read = rep_->index_start_offset |
||||
+ (Rep::GetInternalKeyLength() + Rep::offset_length) * mid; |
||||
Status s = rep_->file->Read(offset_to_read, Rep::GetInternalKeyLength(), |
||||
&tmp_slice, key_chars); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
InternalKeyComparator ikc(rep_->options.comparator); |
||||
int compare_result = ikc.Compare(tmp_slice, target); |
||||
|
||||
if (compare_result < 0) { |
||||
if (left == right) { |
||||
target_offset = right + 1; |
||||
break; |
||||
} |
||||
left = mid; |
||||
} else { |
||||
if (left == right) { |
||||
target_offset = left; |
||||
break; |
||||
} |
||||
right = mid - 1; |
||||
} |
||||
} |
||||
|
||||
if (target_offset >= (uint32_t) rep_->num_entries) { |
||||
*offset = rep_->index_start_offset; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
char value_offset_chars[Rep::offset_length]; |
||||
|
||||
int64_t offset_for_value_offset = rep_->index_start_offset |
||||
+ (Rep::GetInternalKeyLength() + Rep::offset_length) * target_offset |
||||
+ Rep::GetInternalKeyLength(); |
||||
Status s = rep_->file->Read(offset_for_value_offset, Rep::offset_length, |
||||
&tmp_slice, value_offset_chars); |
||||
if (s.ok()) { |
||||
*offset = DecodeFixed64(value_offset_chars); |
||||
} |
||||
return s; |
||||
} |
||||
|
||||
Status SimpleTableReader::Get(const ReadOptions& options, const Slice& k, |
||||
void* arg, |
||||
bool (*saver)(void*, const ParsedInternalKey&, |
||||
const Slice&), |
||||
void (*mark_key_may_exist)(void*)) { |
||||
Status s; |
||||
SimpleTableIterator* iter = new SimpleTableIterator(this); |
||||
for (iter->Seek(k); iter->Valid(); iter->Next()) { |
||||
ParsedInternalKey parsed_key; |
||||
if (!ParseInternalKey(iter->key(), &parsed_key)) { |
||||
return Status::Corruption(Slice()); |
||||
} |
||||
|
||||
if (!(*saver)(arg, parsed_key, iter->value())) { |
||||
break; |
||||
} |
||||
} |
||||
s = iter->status(); |
||||
delete iter; |
||||
return s; |
||||
} |
||||
|
||||
uint64_t SimpleTableReader::ApproximateOffsetOf(const Slice& key) { |
||||
return 0; |
||||
} |
||||
|
||||
SimpleTableIterator::SimpleTableIterator(SimpleTableReader* table) : |
||||
table_(table) { |
||||
key_str_ = new char[SimpleTableReader::Rep::GetInternalKeyLength()]; |
||||
value_str_len_ = -1; |
||||
SeekToFirst(); |
||||
} |
||||
|
||||
SimpleTableIterator::~SimpleTableIterator() { |
||||
delete[] key_str_; |
||||
if (value_str_len_ >= 0) { |
||||
delete[] value_str_; |
||||
} |
||||
} |
||||
|
||||
bool SimpleTableIterator::Valid() const { |
||||
return offset_ < table_->rep_->index_start_offset; |
||||
} |
||||
|
||||
void SimpleTableIterator::SeekToFirst() { |
||||
next_offset_ = 0; |
||||
Next(); |
||||
} |
||||
|
||||
void SimpleTableIterator::SeekToLast() { |
||||
assert(false); |
||||
} |
||||
|
||||
void SimpleTableIterator::Seek(const Slice& target) { |
||||
Status s = table_->GetOffset(target, &next_offset_); |
||||
if (!s.ok()) { |
||||
status_ = s; |
||||
} |
||||
Next(); |
||||
} |
||||
|
||||
void SimpleTableIterator::Next() { |
||||
offset_ = next_offset_; |
||||
if (offset_ >= table_->rep_->index_start_offset) { |
||||
return; |
||||
} |
||||
Slice result; |
||||
int internal_key_size = SimpleTableReader::Rep::GetInternalKeyLength(); |
||||
|
||||
Status s = table_->rep_->file->Read(next_offset_, internal_key_size, &result, |
||||
key_str_); |
||||
next_offset_ += internal_key_size; |
||||
key_ = result; |
||||
|
||||
Slice value_size_slice; |
||||
s = table_->rep_->file->Read(next_offset_, 4, &value_size_slice, tmp_str_); |
||||
next_offset_ += 4; |
||||
uint32_t value_size = DecodeFixed32(tmp_str_); |
||||
|
||||
Slice value_slice; |
||||
if ((int) value_size > value_str_len_) { |
||||
if (value_str_len_ >= 0) { |
||||
delete[] value_str_; |
||||
} |
||||
value_str_ = new char[value_size]; |
||||
value_str_len_ = value_size; |
||||
} |
||||
s = table_->rep_->file->Read(next_offset_, value_size, &value_slice, |
||||
value_str_); |
||||
next_offset_ += value_size; |
||||
value_ = value_slice; |
||||
} |
||||
|
||||
void SimpleTableIterator::Prev() { |
||||
assert(false); |
||||
} |
||||
|
||||
Slice SimpleTableIterator::key() const { |
||||
Log(table_->rep_->options.info_log, "key!!!!"); |
||||
return key_; |
||||
} |
||||
|
||||
Slice SimpleTableIterator::value() const { |
||||
return value_; |
||||
} |
||||
|
||||
Status SimpleTableIterator::status() const { |
||||
return status_; |
||||
} |
||||
|
||||
class SimpleTableBuilder: public TableBuilder { |
||||
public: |
||||
// Create a builder that will store the contents of the table it is
|
||||
// building in *file. Does not close the file. It is up to the
|
||||
// caller to close the file after calling Finish(). The output file
|
||||
// will be part of level specified by 'level'. A value of -1 means
|
||||
// that the caller does not know which level the output file will reside.
|
||||
SimpleTableBuilder(const Options& options, WritableFile* file, |
||||
CompressionType compression_type); |
||||
|
||||
// REQUIRES: Either Finish() or Abandon() has been called.
|
||||
~SimpleTableBuilder(); |
||||
|
||||
// Add key,value to the table being constructed.
|
||||
// REQUIRES: key is after any previously added key according to comparator.
|
||||
// REQUIRES: Finish(), Abandon() have not been called
|
||||
void Add(const Slice& key, const Slice& value) override; |
||||
|
||||
// Return non-ok iff some error has been detected.
|
||||
Status status() const override; |
||||
|
||||
// Finish building the table. Stops using the file passed to the
|
||||
// constructor after this function returns.
|
||||
// REQUIRES: Finish(), Abandon() have not been called
|
||||
Status Finish() override; |
||||
|
||||
// Indicate that the contents of this builder should be abandoned. Stops
|
||||
// using the file passed to the constructor after this function returns.
|
||||
// If the caller is not going to call Finish(), it must call Abandon()
|
||||
// before destroying this builder.
|
||||
// REQUIRES: Finish(), Abandon() have not been called
|
||||
void Abandon() override; |
||||
|
||||
// Number of calls to Add() so far.
|
||||
uint64_t NumEntries() const override; |
||||
|
||||
// Size of the file generated so far. If invoked after a successful
|
||||
// Finish() call, returns the size of the final generated file.
|
||||
uint64_t FileSize() const override; |
||||
|
||||
private: |
||||
struct Rep; |
||||
Rep* rep_; |
||||
|
||||
// No copying allowed
|
||||
SimpleTableBuilder(const SimpleTableBuilder&) = delete; |
||||
void operator=(const SimpleTableBuilder&) = delete; |
||||
}; |
||||
|
||||
struct SimpleTableBuilder::Rep { |
||||
Options options; |
||||
WritableFile* file; |
||||
uint64_t offset = 0; |
||||
Status status; |
||||
|
||||
uint64_t num_entries = 0; |
||||
|
||||
bool closed = false; // Either Finish() or Abandon() has been called.
|
||||
|
||||
const static int user_key_size = 16; |
||||
const static int offset_length = 8; |
||||
const static int key_footer_len = 8; |
||||
|
||||
static int GetInternalKeyLength() { |
||||
return user_key_size + key_footer_len; |
||||
} |
||||
|
||||
std::string index; |
||||
|
||||
Rep(const Options& opt, WritableFile* f) : |
||||
options(opt), file(f) { |
||||
} |
||||
~Rep() { |
||||
} |
||||
}; |
||||
|
||||
SimpleTableBuilder::SimpleTableBuilder(const Options& options, |
||||
WritableFile* file, |
||||
CompressionType compression_type) : |
||||
rep_(new SimpleTableBuilder::Rep(options, file)) { |
||||
} |
||||
|
||||
SimpleTableBuilder::~SimpleTableBuilder() { |
||||
delete (rep_); |
||||
} |
||||
|
||||
void SimpleTableBuilder::Add(const Slice& key, const Slice& value) { |
||||
assert((int ) key.size() == Rep::GetInternalKeyLength()); |
||||
|
||||
// Update index
|
||||
rep_->index.append(key.data(), key.size()); |
||||
PutFixed64(&(rep_->index), rep_->offset); |
||||
|
||||
// Write key-value pair
|
||||
rep_->file->Append(key); |
||||
rep_->offset += Rep::GetInternalKeyLength(); |
||||
|
||||
std::string size; |
||||
int value_size = value.size(); |
||||
PutFixed32(&size, value_size); |
||||
Slice sizeSlice(size); |
||||
rep_->file->Append(sizeSlice); |
||||
rep_->file->Append(value); |
||||
rep_->offset += value_size + 4; |
||||
|
||||
rep_->num_entries++; |
||||
} |
||||
|
||||
Status SimpleTableBuilder::status() const { |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status SimpleTableBuilder::Finish() { |
||||
Rep* r = rep_; |
||||
assert(!r->closed); |
||||
r->closed = true; |
||||
|
||||
uint64_t index_offset = rep_->offset; |
||||
Slice index_slice(rep_->index); |
||||
rep_->file->Append(index_slice); |
||||
rep_->offset += index_slice.size(); |
||||
|
||||
std::string index_offset_str; |
||||
PutFixed64(&index_offset_str, index_offset); |
||||
Slice foot_slice(index_offset_str); |
||||
rep_->file->Append(foot_slice); |
||||
rep_->offset += foot_slice.size(); |
||||
|
||||
return Status::OK(); |
||||
} |
||||
|
||||
void SimpleTableBuilder::Abandon() { |
||||
rep_->closed = true; |
||||
} |
||||
|
||||
uint64_t SimpleTableBuilder::NumEntries() const { |
||||
return rep_->num_entries; |
||||
} |
||||
|
||||
uint64_t SimpleTableBuilder::FileSize() const { |
||||
return rep_->offset; |
||||
} |
||||
|
||||
class SimpleTableFactory: public TableFactory { |
||||
public: |
||||
~SimpleTableFactory() { |
||||
} |
||||
SimpleTableFactory() { |
||||
} |
||||
const char* Name() const override { |
||||
return "SimpleTable"; |
||||
} |
||||
Status NewTableReader(const Options& options, const EnvOptions& soptions, |
||||
const InternalKeyComparator& internal_key, |
||||
unique_ptr<RandomAccessFile>&& file, uint64_t file_size, |
||||
unique_ptr<TableReader>* table_reader) const; |
||||
|
||||
TableBuilder* NewTableBuilder(const Options& options, |
||||
const InternalKeyComparator& internal_key, |
||||
WritableFile* file, |
||||
CompressionType compression_type) const; |
||||
|
||||
virtual Status SanitizeDBOptions(DBOptions* db_opts) const override { |
||||
return Status::OK(); |
||||
} |
||||
|
||||
virtual std::string GetPrintableTableOptions() const override { |
||||
return std::string(); |
||||
} |
||||
}; |
||||
|
||||
Status SimpleTableFactory::NewTableReader( |
||||
const Options& options, const EnvOptions& soptions, |
||||
const InternalKeyComparator& internal_key, |
||||
unique_ptr<RandomAccessFile>&& file, uint64_t file_size, |
||||
unique_ptr<TableReader>* table_reader) const { |
||||
|
||||
return SimpleTableReader::Open(options, soptions, std::move(file), file_size, |
||||
table_reader); |
||||
} |
||||
|
||||
TableBuilder* SimpleTableFactory::NewTableBuilder( |
||||
const Options& options, const InternalKeyComparator& internal_key, |
||||
WritableFile* file, CompressionType compression_type) const { |
||||
return new SimpleTableBuilder(options, file, compression_type); |
||||
} |
||||
|
||||
class SimpleTableDBTest { |
||||
protected: |
||||
public: |
||||
std::string dbname_; |
||||
Env* env_; |
||||
DB* db_; |
||||
|
||||
Options last_options_; |
||||
|
||||
SimpleTableDBTest() : |
||||
env_(Env::Default()) { |
||||
dbname_ = test::TmpDir() + "/simple_table_db_test"; |
||||
ASSERT_OK(DestroyDB(dbname_, Options())); |
||||
db_ = nullptr; |
||||
Reopen(); |
||||
} |
||||
|
||||
~SimpleTableDBTest() { |
||||
delete db_; |
||||
ASSERT_OK(DestroyDB(dbname_, Options())); |
||||
} |
||||
|
||||
// Return the current option configuration.
|
||||
Options CurrentOptions() { |
||||
Options options; |
||||
options.table_factory.reset(new SimpleTableFactory()); |
||||
return options; |
||||
} |
||||
|
||||
DBImpl* dbfull() { |
||||
return reinterpret_cast<DBImpl*>(db_); |
||||
} |
||||
|
||||
void Reopen(Options* options = nullptr) { |
||||
ASSERT_OK(TryReopen(options)); |
||||
} |
||||
|
||||
void Close() { |
||||
delete db_; |
||||
db_ = nullptr; |
||||
} |
||||
|
||||
void DestroyAndReopen(Options* options = nullptr) { |
||||
//Destroy using last options
|
||||
Destroy(&last_options_); |
||||
ASSERT_OK(TryReopen(options)); |
||||
} |
||||
|
||||
void Destroy(Options* options) { |
||||
delete db_; |
||||
db_ = nullptr; |
||||
ASSERT_OK(DestroyDB(dbname_, *options)); |
||||
} |
||||
|
||||
Status PureReopen(Options* options, DB** db) { |
||||
return DB::Open(*options, dbname_, db); |
||||
} |
||||
|
||||
Status TryReopen(Options* options = nullptr) { |
||||
delete db_; |
||||
db_ = nullptr; |
||||
Options opts; |
||||
if (options != nullptr) { |
||||
opts = *options; |
||||
} else { |
||||
opts = CurrentOptions(); |
||||
opts.create_if_missing = true; |
||||
} |
||||
last_options_ = opts; |
||||
|
||||
return DB::Open(opts, dbname_, &db_); |
||||
} |
||||
|
||||
Status Put(const Slice& k, const Slice& v) { |
||||
return db_->Put(WriteOptions(), k, v); |
||||
} |
||||
|
||||
Status Delete(const std::string& k) { |
||||
return db_->Delete(WriteOptions(), k); |
||||
} |
||||
|
||||
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { |
||||
ReadOptions options; |
||||
options.snapshot = snapshot; |
||||
std::string result; |
||||
Status s = db_->Get(options, k, &result); |
||||
if (s.IsNotFound()) { |
||||
result = "NOT_FOUND"; |
||||
} else if (!s.ok()) { |
||||
result = s.ToString(); |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
|
||||
int NumTableFilesAtLevel(int level) { |
||||
std::string property; |
||||
ASSERT_TRUE( |
||||
db_->GetProperty("rocksdb.num-files-at-level" + NumberToString(level), |
||||
&property)); |
||||
return atoi(property.c_str()); |
||||
} |
||||
|
||||
// Return spread of files per level
|
||||
std::string FilesPerLevel() { |
||||
std::string result; |
||||
int last_non_zero_offset = 0; |
||||
for (int level = 0; level < db_->NumberLevels(); level++) { |
||||
int f = NumTableFilesAtLevel(level); |
||||
char buf[100]; |
||||
snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); |
||||
result += buf; |
||||
if (f > 0) { |
||||
last_non_zero_offset = result.size(); |
||||
} |
||||
} |
||||
result.resize(last_non_zero_offset); |
||||
return result; |
||||
} |
||||
|
||||
std::string IterStatus(Iterator* iter) { |
||||
std::string result; |
||||
if (iter->Valid()) { |
||||
result = iter->key().ToString() + "->" + iter->value().ToString(); |
||||
} else { |
||||
result = "(invalid)"; |
||||
} |
||||
return result; |
||||
} |
||||
}; |
||||
|
||||
TEST(SimpleTableDBTest, Empty) { |
||||
ASSERT_TRUE(db_ != nullptr); |
||||
ASSERT_EQ("NOT_FOUND", Get("0000000000000foo")); |
||||
} |
||||
|
||||
TEST(SimpleTableDBTest, ReadWrite) { |
||||
ASSERT_OK(Put("0000000000000foo", "v1")); |
||||
ASSERT_EQ("v1", Get("0000000000000foo")); |
||||
ASSERT_OK(Put("0000000000000bar", "v2")); |
||||
ASSERT_OK(Put("0000000000000foo", "v3")); |
||||
ASSERT_EQ("v3", Get("0000000000000foo")); |
||||
ASSERT_EQ("v2", Get("0000000000000bar")); |
||||
} |
||||
|
||||
TEST(SimpleTableDBTest, Flush) { |
||||
ASSERT_OK(Put("0000000000000foo", "v1")); |
||||
ASSERT_OK(Put("0000000000000bar", "v2")); |
||||
ASSERT_OK(Put("0000000000000foo", "v3")); |
||||
dbfull()->TEST_FlushMemTable(); |
||||
ASSERT_EQ("v3", Get("0000000000000foo")); |
||||
ASSERT_EQ("v2", Get("0000000000000bar")); |
||||
} |
||||
|
||||
TEST(SimpleTableDBTest, Flush2) { |
||||
ASSERT_OK(Put("0000000000000bar", "b")); |
||||
ASSERT_OK(Put("0000000000000foo", "v1")); |
||||
dbfull()->TEST_FlushMemTable(); |
||||
|
||||
ASSERT_OK(Put("0000000000000foo", "v2")); |
||||
dbfull()->TEST_FlushMemTable(); |
||||
ASSERT_EQ("v2", Get("0000000000000foo")); |
||||
|
||||
ASSERT_OK(Put("0000000000000eee", "v3")); |
||||
dbfull()->TEST_FlushMemTable(); |
||||
ASSERT_EQ("v3", Get("0000000000000eee")); |
||||
|
||||
ASSERT_OK(Delete("0000000000000bar")); |
||||
dbfull()->TEST_FlushMemTable(); |
||||
ASSERT_EQ("NOT_FOUND", Get("0000000000000bar")); |
||||
|
||||
ASSERT_OK(Put("0000000000000eee", "v5")); |
||||
dbfull()->TEST_FlushMemTable(); |
||||
ASSERT_EQ("v5", Get("0000000000000eee")); |
||||
} |
||||
|
||||
static std::string Key(int i) { |
||||
char buf[100]; |
||||
snprintf(buf, sizeof(buf), "key_______%06d", i); |
||||
return std::string(buf); |
||||
} |
||||
|
||||
static std::string RandomString(Random* rnd, int len) { |
||||
std::string r; |
||||
test::RandomString(rnd, len, &r); |
||||
return r; |
||||
} |
||||
|
||||
TEST(SimpleTableDBTest, CompactionTrigger) { |
||||
Options options = CurrentOptions(); |
||||
options.write_buffer_size = 100 << 10; //100KB
|
||||
options.num_levels = 3; |
||||
options.max_mem_compaction_level = 0; |
||||
options.level0_file_num_compaction_trigger = 3; |
||||
Reopen(&options); |
||||
|
||||
Random rnd(301); |
||||
|
||||
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1; |
||||
num++) { |
||||
std::vector<std::string> values; |
||||
// Write 120KB (12 values, each 10K)
|
||||
for (int i = 0; i < 12; i++) { |
||||
values.push_back(RandomString(&rnd, 10000)); |
||||
ASSERT_OK(Put(Key(i), values[i])); |
||||
} |
||||
dbfull()->TEST_WaitForFlushMemTable(); |
||||
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1); |
||||
} |
||||
|
||||
//generate one more file in level-0, and should trigger level-0 compaction
|
||||
std::vector<std::string> values; |
||||
for (int i = 0; i < 12; i++) { |
||||
values.push_back(RandomString(&rnd, 10000)); |
||||
ASSERT_OK(Put(Key(i), values[i])); |
||||
} |
||||
dbfull()->TEST_WaitForCompact(); |
||||
|
||||
ASSERT_EQ(NumTableFilesAtLevel(0), 0); |
||||
ASSERT_EQ(NumTableFilesAtLevel(1), 1); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
return rocksdb::test::RunAllTests(); |
||||
} |
||||
@ -0,0 +1,330 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/version_builder.h" |
||||
|
||||
#ifndef __STDC_FORMAT_MACROS |
||||
#define __STDC_FORMAT_MACROS |
||||
#endif |
||||
|
||||
#include <inttypes.h> |
||||
#include <algorithm> |
||||
#include <set> |
||||
#include <unordered_map> |
||||
#include <unordered_set> |
||||
#include <vector> |
||||
|
||||
#include "db/dbformat.h" |
||||
#include "db/table_cache.h" |
||||
#include "db/version_set.h" |
||||
#include "table/table_reader.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b) { |
||||
if (a->smallest_seqno != b->smallest_seqno) { |
||||
return a->smallest_seqno > b->smallest_seqno; |
||||
} |
||||
if (a->largest_seqno != b->largest_seqno) { |
||||
return a->largest_seqno > b->largest_seqno; |
||||
} |
||||
// Break ties by file number
|
||||
return a->fd.GetNumber() > b->fd.GetNumber(); |
||||
} |
||||
|
||||
namespace { |
||||
bool BySmallestKey(FileMetaData* a, FileMetaData* b, |
||||
const InternalKeyComparator* cmp) { |
||||
int r = cmp->Compare(a->smallest, b->smallest); |
||||
if (r != 0) { |
||||
return (r < 0); |
||||
} |
||||
// Break ties by file number
|
||||
return (a->fd.GetNumber() < b->fd.GetNumber()); |
||||
} |
||||
} // namespace
|
||||
|
||||
class VersionBuilder::Rep { |
||||
private: |
||||
// Helper to sort files_ in v
|
||||
// kLevel0 -- NewestFirstBySeqNo
|
||||
// kLevelNon0 -- BySmallestKey
|
||||
struct FileComparator { |
||||
enum SortMethod { kLevel0 = 0, kLevelNon0 = 1, } sort_method; |
||||
const InternalKeyComparator* internal_comparator; |
||||
|
||||
bool operator()(FileMetaData* f1, FileMetaData* f2) const { |
||||
switch (sort_method) { |
||||
case kLevel0: |
||||
return NewestFirstBySeqNo(f1, f2); |
||||
case kLevelNon0: |
||||
return BySmallestKey(f1, f2, internal_comparator); |
||||
} |
||||
assert(false); |
||||
return false; |
||||
} |
||||
}; |
||||
|
||||
struct LevelState { |
||||
std::unordered_set<uint64_t> deleted_files; |
||||
// Map from file number to file meta data.
|
||||
std::unordered_map<uint64_t, FileMetaData*> added_files; |
||||
}; |
||||
|
||||
const EnvOptions& env_options_; |
||||
TableCache* table_cache_; |
||||
VersionStorageInfo* base_vstorage_; |
||||
LevelState* levels_; |
||||
FileComparator level_zero_cmp_; |
||||
FileComparator level_nonzero_cmp_; |
||||
|
||||
public: |
||||
Rep(const EnvOptions& env_options, TableCache* table_cache, |
||||
VersionStorageInfo* base_vstorage) |
||||
: env_options_(env_options), |
||||
table_cache_(table_cache), |
||||
base_vstorage_(base_vstorage) { |
||||
levels_ = new LevelState[base_vstorage_->num_levels()]; |
||||
level_zero_cmp_.sort_method = FileComparator::kLevel0; |
||||
level_nonzero_cmp_.sort_method = FileComparator::kLevelNon0; |
||||
level_nonzero_cmp_.internal_comparator = |
||||
base_vstorage_->InternalComparator(); |
||||
} |
||||
|
||||
~Rep() { |
||||
for (int level = 0; level < base_vstorage_->num_levels(); level++) { |
||||
const auto& added = levels_[level].added_files; |
||||
for (auto& pair : added) { |
||||
UnrefFile(pair.second); |
||||
} |
||||
} |
||||
|
||||
delete[] levels_; |
||||
} |
||||
|
||||
void UnrefFile(FileMetaData* f) { |
||||
f->refs--; |
||||
if (f->refs <= 0) { |
||||
if (f->table_reader_handle) { |
||||
assert(table_cache_ != nullptr); |
||||
table_cache_->ReleaseHandle(f->table_reader_handle); |
||||
f->table_reader_handle = nullptr; |
||||
} |
||||
delete f; |
||||
} |
||||
} |
||||
|
||||
void CheckConsistency(VersionStorageInfo* vstorage) { |
||||
#ifndef NDEBUG |
||||
// make sure the files are sorted correctly
|
||||
for (int level = 0; level < vstorage->num_levels(); level++) { |
||||
auto& level_files = vstorage->LevelFiles(level); |
||||
for (size_t i = 1; i < level_files.size(); i++) { |
||||
auto f1 = level_files[i - 1]; |
||||
auto f2 = level_files[i]; |
||||
if (level == 0) { |
||||
assert(level_zero_cmp_(f1, f2)); |
||||
assert(f1->largest_seqno > f2->largest_seqno); |
||||
} else { |
||||
assert(level_nonzero_cmp_(f1, f2)); |
||||
|
||||
// Make sure there is no overlap in levels > 0
|
||||
if (vstorage->InternalComparator()->Compare(f1->largest, |
||||
f2->smallest) >= 0) { |
||||
fprintf(stderr, "overlapping ranges in same level %s vs. %s\n", |
||||
(f1->largest).DebugString().c_str(), |
||||
(f2->smallest).DebugString().c_str()); |
||||
abort(); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
#endif |
||||
} |
||||
|
||||
void CheckConsistencyForDeletes(VersionEdit* edit, uint64_t number, |
||||
int level) { |
||||
#ifndef NDEBUG |
||||
// a file to be deleted better exist in the previous version
|
||||
bool found = false; |
||||
for (int l = 0; !found && l < base_vstorage_->num_levels(); l++) { |
||||
const std::vector<FileMetaData*>& base_files = |
||||
base_vstorage_->LevelFiles(l); |
||||
for (unsigned int i = 0; i < base_files.size(); i++) { |
||||
FileMetaData* f = base_files[i]; |
||||
if (f->fd.GetNumber() == number) { |
||||
found = true; |
||||
break; |
||||
} |
||||
} |
||||
} |
||||
// if the file did not exist in the previous version, then it
|
||||
// is possibly moved from lower level to higher level in current
|
||||
// version
|
||||
for (int l = level + 1; !found && l < base_vstorage_->num_levels(); l++) { |
||||
auto& level_added = levels_[l].added_files; |
||||
auto got = level_added.find(number); |
||||
if (got != level_added.end()) { |
||||
found = true; |
||||
break; |
||||
} |
||||
} |
||||
|
||||
// maybe this file was added in a previous edit that was Applied
|
||||
if (!found) { |
||||
auto& level_added = levels_[level].added_files; |
||||
auto got = level_added.find(number); |
||||
if (got != level_added.end()) { |
||||
found = true; |
||||
} |
||||
} |
||||
if (!found) { |
||||
fprintf(stderr, "not found %" PRIu64 "\n", number); |
||||
} |
||||
assert(found); |
||||
#endif |
||||
} |
||||
|
||||
// Apply all of the edits in *edit to the current state.
|
||||
void Apply(VersionEdit* edit) { |
||||
CheckConsistency(base_vstorage_); |
||||
|
||||
// Delete files
|
||||
const VersionEdit::DeletedFileSet& del = edit->GetDeletedFiles(); |
||||
for (const auto& del_file : del) { |
||||
const auto level = del_file.first; |
||||
const auto number = del_file.second; |
||||
levels_[level].deleted_files.insert(number); |
||||
CheckConsistencyForDeletes(edit, number, level); |
||||
|
||||
auto exising = levels_[level].added_files.find(number); |
||||
if (exising != levels_[level].added_files.end()) { |
||||
UnrefFile(exising->second); |
||||
levels_[level].added_files.erase(number); |
||||
} |
||||
} |
||||
|
||||
// Add new files
|
||||
for (const auto& new_file : edit->GetNewFiles()) { |
||||
const int level = new_file.first; |
||||
FileMetaData* f = new FileMetaData(new_file.second); |
||||
f->refs = 1; |
||||
|
||||
assert(levels_[level].added_files.find(f->fd.GetNumber()) == |
||||
levels_[level].added_files.end()); |
||||
levels_[level].deleted_files.erase(f->fd.GetNumber()); |
||||
levels_[level].added_files[f->fd.GetNumber()] = f; |
||||
} |
||||
} |
||||
|
||||
// Save the current state in *v.
|
||||
void SaveTo(VersionStorageInfo* vstorage) { |
||||
CheckConsistency(base_vstorage_); |
||||
CheckConsistency(vstorage); |
||||
|
||||
for (int level = 0; level < base_vstorage_->num_levels(); level++) { |
||||
const auto& cmp = (level == 0) ? level_zero_cmp_ : level_nonzero_cmp_; |
||||
// Merge the set of added files with the set of pre-existing files.
|
||||
// Drop any deleted files. Store the result in *v.
|
||||
const auto& base_files = base_vstorage_->LevelFiles(level); |
||||
auto base_iter = base_files.begin(); |
||||
auto base_end = base_files.end(); |
||||
const auto& unordered_added_files = levels_[level].added_files; |
||||
vstorage->Reserve(level, |
||||
base_files.size() + unordered_added_files.size()); |
||||
|
||||
// Sort added files for the level.
|
||||
std::vector<FileMetaData*> added_files; |
||||
added_files.reserve(unordered_added_files.size()); |
||||
for (const auto& pair : unordered_added_files) { |
||||
added_files.push_back(pair.second); |
||||
} |
||||
std::sort(added_files.begin(), added_files.end(), cmp); |
||||
|
||||
#ifndef NDEBUG |
||||
FileMetaData* prev_file = nullptr; |
||||
#endif |
||||
|
||||
for (const auto& added : added_files) { |
||||
#ifndef NDEBUG |
||||
if (level > 0 && prev_file != nullptr) { |
||||
assert(base_vstorage_->InternalComparator()->Compare( |
||||
prev_file->smallest, added->smallest) <= 0); |
||||
} |
||||
prev_file = added; |
||||
#endif |
||||
|
||||
// Add all smaller files listed in base_
|
||||
for (auto bpos = std::upper_bound(base_iter, base_end, added, cmp); |
||||
base_iter != bpos; ++base_iter) { |
||||
MaybeAddFile(vstorage, level, *base_iter); |
||||
} |
||||
|
||||
MaybeAddFile(vstorage, level, added); |
||||
} |
||||
|
||||
// Add remaining base files
|
||||
for (; base_iter != base_end; ++base_iter) { |
||||
MaybeAddFile(vstorage, level, *base_iter); |
||||
} |
||||
} |
||||
|
||||
CheckConsistency(vstorage); |
||||
} |
||||
|
||||
void LoadTableHandlers() { |
||||
assert(table_cache_ != nullptr); |
||||
for (int level = 0; level < base_vstorage_->num_levels(); level++) { |
||||
for (auto& file_meta_pair : levels_[level].added_files) { |
||||
auto* file_meta = file_meta_pair.second; |
||||
assert(!file_meta->table_reader_handle); |
||||
table_cache_->FindTable( |
||||
env_options_, *(base_vstorage_->InternalComparator()), |
||||
file_meta->fd, &file_meta->table_reader_handle, false); |
||||
if (file_meta->table_reader_handle != nullptr) { |
||||
// Load table_reader
|
||||
file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle( |
||||
file_meta->table_reader_handle); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f) { |
||||
if (levels_[level].deleted_files.count(f->fd.GetNumber()) > 0) { |
||||
// File is deleted: do nothing
|
||||
} else { |
||||
vstorage->AddFile(level, f); |
||||
} |
||||
} |
||||
}; |
||||
|
||||
VersionBuilder::VersionBuilder(const EnvOptions& env_options, |
||||
TableCache* table_cache, |
||||
VersionStorageInfo* base_vstorage) |
||||
: rep_(new Rep(env_options, table_cache, base_vstorage)) {} |
||||
VersionBuilder::~VersionBuilder() { delete rep_; } |
||||
void VersionBuilder::CheckConsistency(VersionStorageInfo* vstorage) { |
||||
rep_->CheckConsistency(vstorage); |
||||
} |
||||
void VersionBuilder::CheckConsistencyForDeletes(VersionEdit* edit, |
||||
uint64_t number, int level) { |
||||
rep_->CheckConsistencyForDeletes(edit, number, level); |
||||
} |
||||
void VersionBuilder::Apply(VersionEdit* edit) { rep_->Apply(edit); } |
||||
void VersionBuilder::SaveTo(VersionStorageInfo* vstorage) { |
||||
rep_->SaveTo(vstorage); |
||||
} |
||||
void VersionBuilder::LoadTableHandlers() { rep_->LoadTableHandlers(); } |
||||
void VersionBuilder::MaybeAddFile(VersionStorageInfo* vstorage, int level, |
||||
FileMetaData* f) { |
||||
rep_->MaybeAddFile(vstorage, level, f); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,42 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
#pragma once |
||||
#include "rocksdb/env.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class TableCache; |
||||
class VersionStorageInfo; |
||||
class VersionEdit; |
||||
struct FileMetaData; |
||||
|
||||
// A helper class so we can efficiently apply a whole sequence
|
||||
// of edits to a particular state without creating intermediate
|
||||
// Versions that contain full copies of the intermediate state.
|
||||
class VersionBuilder { |
||||
public: |
||||
VersionBuilder(const EnvOptions& env_options, TableCache* table_cache, |
||||
VersionStorageInfo* base_vstorage); |
||||
~VersionBuilder(); |
||||
void CheckConsistency(VersionStorageInfo* vstorage); |
||||
void CheckConsistencyForDeletes(VersionEdit* edit, uint64_t number, |
||||
int level); |
||||
void Apply(VersionEdit* edit); |
||||
void SaveTo(VersionStorageInfo* vstorage); |
||||
void LoadTableHandlers(); |
||||
void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f); |
||||
|
||||
private: |
||||
class Rep; |
||||
Rep* rep_; |
||||
}; |
||||
|
||||
extern bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b); |
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,229 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <string> |
||||
#include "db/version_edit.h" |
||||
#include "db/version_set.h" |
||||
#include "util/logging.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class VersionBuilderTest { |
||||
public: |
||||
const Comparator* ucmp_; |
||||
InternalKeyComparator icmp_; |
||||
Options options_; |
||||
ImmutableCFOptions ioptions_; |
||||
MutableCFOptions mutable_cf_options_; |
||||
VersionStorageInfo vstorage_; |
||||
uint32_t file_num_; |
||||
CompactionOptionsFIFO fifo_options_; |
||||
std::vector<uint64_t> size_being_compacted_; |
||||
|
||||
VersionBuilderTest() |
||||
: ucmp_(BytewiseComparator()), |
||||
icmp_(ucmp_), |
||||
ioptions_(options_), |
||||
mutable_cf_options_(options_, ioptions_), |
||||
vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel, |
||||
nullptr), |
||||
file_num_(1) { |
||||
mutable_cf_options_.RefreshDerivedOptions(ioptions_); |
||||
size_being_compacted_.resize(options_.num_levels); |
||||
} |
||||
|
||||
~VersionBuilderTest() { |
||||
for (int i = 0; i < vstorage_.num_levels(); i++) { |
||||
for (auto* f : vstorage_.LevelFiles(i)) { |
||||
if (--f->refs == 0) { |
||||
delete f; |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
InternalKey GetInternalKey(const char* ukey, |
||||
SequenceNumber smallest_seq = 100) { |
||||
return InternalKey(ukey, smallest_seq, kTypeValue); |
||||
} |
||||
|
||||
void Add(int level, uint32_t file_number, const char* smallest, |
||||
const char* largest, uint64_t file_size = 0, uint32_t path_id = 0, |
||||
SequenceNumber smallest_seq = 100, |
||||
SequenceNumber largest_seq = 100, |
||||
uint64_t num_entries = 0, uint64_t num_deletions = 0, |
||||
bool sampled = false) { |
||||
assert(level < vstorage_.num_levels()); |
||||
FileMetaData* f = new FileMetaData; |
||||
f->fd = FileDescriptor(file_number, path_id, file_size); |
||||
f->smallest = GetInternalKey(smallest, smallest_seq); |
||||
f->largest = GetInternalKey(largest, largest_seq); |
||||
f->compensated_file_size = file_size; |
||||
f->refs = 0; |
||||
f->num_entries = num_entries; |
||||
f->num_deletions = num_deletions; |
||||
vstorage_.AddFile(level, f); |
||||
if (sampled) { |
||||
f->init_stats_from_file = true; |
||||
vstorage_.UpdateAccumulatedStats(f); |
||||
} |
||||
} |
||||
|
||||
void UpdateVersionStorageInfo() { |
||||
vstorage_.UpdateFilesBySize(); |
||||
vstorage_.UpdateNumNonEmptyLevels(); |
||||
vstorage_.GenerateFileIndexer(); |
||||
vstorage_.GenerateLevelFilesBrief(); |
||||
vstorage_.SetFinalized(); |
||||
} |
||||
}; |
||||
|
||||
TEST(VersionBuilderTest, ApplyAndSaveTo) { |
||||
Add(0, 1U, "150", "200", 100U); |
||||
// Level 1 score 1.2
|
||||
Add(1, 66U, "150", "200", 100U); |
||||
Add(1, 88U, "201", "300", 100U); |
||||
// Level 2 score 1.8. File 7 is the largest. Should be picked
|
||||
Add(2, 6U, "150", "179", 100U); |
||||
Add(2, 7U, "180", "220", 100U); |
||||
Add(2, 8U, "221", "300", 100U); |
||||
// Level 3 score slightly larger than 1
|
||||
Add(3, 26U, "150", "170", 100U); |
||||
Add(3, 27U, "171", "179", 100U); |
||||
Add(3, 28U, "191", "220", 100U); |
||||
Add(3, 29U, "221", "300", 100U); |
||||
UpdateVersionStorageInfo(); |
||||
|
||||
VersionEdit version_edit; |
||||
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"), |
||||
GetInternalKey("350"), 200, 200); |
||||
version_edit.DeleteFile(3, 27U); |
||||
|
||||
EnvOptions env_options; |
||||
|
||||
VersionBuilder version_builder(env_options, nullptr, &vstorage_); |
||||
|
||||
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, |
||||
kCompactionStyleLevel, nullptr); |
||||
version_builder.Apply(&version_edit); |
||||
version_builder.SaveTo(&new_vstorage); |
||||
|
||||
ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2)); |
||||
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3)); |
||||
|
||||
for (int i = 0; i < new_vstorage.num_levels(); i++) { |
||||
for (auto* f : new_vstorage.LevelFiles(i)) { |
||||
if (--f->refs == 0) { |
||||
delete f; |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST(VersionBuilderTest, ApplyMultipleAndSaveTo) { |
||||
UpdateVersionStorageInfo(); |
||||
|
||||
VersionEdit version_edit; |
||||
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"), |
||||
GetInternalKey("350"), 200, 200); |
||||
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"), |
||||
GetInternalKey("450"), 200, 200); |
||||
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"), |
||||
GetInternalKey("650"), 200, 200); |
||||
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"), |
||||
GetInternalKey("550"), 200, 200); |
||||
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"), |
||||
GetInternalKey("750"), 200, 200); |
||||
|
||||
EnvOptions env_options; |
||||
|
||||
VersionBuilder version_builder(env_options, nullptr, &vstorage_); |
||||
|
||||
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, |
||||
kCompactionStyleLevel, nullptr); |
||||
version_builder.Apply(&version_edit); |
||||
version_builder.SaveTo(&new_vstorage); |
||||
|
||||
ASSERT_EQ(500U, new_vstorage.NumLevelBytes(2)); |
||||
|
||||
for (int i = 0; i < new_vstorage.num_levels(); i++) { |
||||
for (auto* f : new_vstorage.LevelFiles(i)) { |
||||
if (--f->refs == 0) { |
||||
delete f; |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST(VersionBuilderTest, ApplyDeleteAndSaveTo) { |
||||
UpdateVersionStorageInfo(); |
||||
|
||||
EnvOptions env_options; |
||||
VersionBuilder version_builder(env_options, nullptr, &vstorage_); |
||||
VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels, |
||||
kCompactionStyleLevel, nullptr); |
||||
|
||||
VersionEdit version_edit; |
||||
version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"), |
||||
GetInternalKey("350"), 200, 200); |
||||
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"), |
||||
GetInternalKey("450"), 200, 200); |
||||
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"), |
||||
GetInternalKey("650"), 200, 200); |
||||
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"), |
||||
GetInternalKey("550"), 200, 200); |
||||
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"), |
||||
GetInternalKey("750"), 200, 200); |
||||
version_builder.Apply(&version_edit); |
||||
|
||||
VersionEdit version_edit2; |
||||
version_edit.AddFile(2, 808, 0, 100U, GetInternalKey("901"), |
||||
GetInternalKey("950"), 200, 200); |
||||
version_edit2.DeleteFile(2, 616); |
||||
version_edit2.DeleteFile(2, 636); |
||||
version_edit.AddFile(2, 806, 0, 100U, GetInternalKey("801"), |
||||
GetInternalKey("850"), 200, 200); |
||||
version_builder.Apply(&version_edit2); |
||||
|
||||
version_builder.SaveTo(&new_vstorage); |
||||
|
||||
ASSERT_EQ(300U, new_vstorage.NumLevelBytes(2)); |
||||
|
||||
for (int i = 0; i < new_vstorage.num_levels(); i++) { |
||||
for (auto* f : new_vstorage.LevelFiles(i)) { |
||||
if (--f->refs == 0) { |
||||
delete f; |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST(VersionBuilderTest, EstimatedActiveKeys) { |
||||
const uint32_t kTotalSamples = 20; |
||||
const uint32_t kNumLevels = 5; |
||||
const uint32_t kFilesPerLevel = 8; |
||||
const uint32_t kNumFiles = kNumLevels * kFilesPerLevel; |
||||
const uint32_t kEntriesPerFile = 1000; |
||||
const uint32_t kDeletionsPerFile = 100; |
||||
for (uint32_t i = 0; i < kNumFiles; ++i) { |
||||
Add(static_cast<int>(i / kFilesPerLevel), i + 1, |
||||
ToString((i + 100) * 1000).c_str(), |
||||
ToString((i + 100) * 1000 + 999).c_str(), |
||||
100U, 0, 100, 100, |
||||
kEntriesPerFile, kDeletionsPerFile, |
||||
(i < kTotalSamples)); |
||||
} |
||||
// minus 2X for the number of deletion entries because:
|
||||
// 1x for deletion entry does not count as a data entry.
|
||||
// 1x for each deletion entry will actually remove one data entry.
|
||||
ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(), |
||||
(kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue