AIX and Solaris Sparc Support

Summary:
Replacement of #2147

The change was squashed due to a lot of conflicts.
Closes https://github.com/facebook/rocksdb/pull/2194

Differential Revision: D4929799

Pulled By: siying

fbshipit-source-id: 5cd49c254737a1d5ac13f3c035f128e86524c581
main
Tomas Kolda 8 years ago committed by Facebook Github Bot
parent afff9951e2
commit 04d58970cb
  1. 15
      CMakeLists.txt
  2. 27
      INSTALL.md
  3. 78
      Makefile
  4. 25
      build_tools/build_detect_platform
  5. 6
      db/cuckoo_table_db_test.cc
  6. 2
      db/db_sst_test.cc
  7. 4
      db/db_test2.cc
  8. 2
      db/external_sst_file_ingestion_job.cc
  9. 1
      db/internal_stats.h
  10. 8
      db/merge_test.cc
  11. 15
      db/perf_context_test.cc
  12. 68
      db/prefix_test.cc
  13. 14
      env/env_chroot.cc
  14. 28
      env/env_posix.cc
  15. 10
      env/env_test.cc
  16. 10
      env/io_posix.cc
  17. 8
      env/io_posix.h
  18. 64
      java/CMakeLists.txt
  19. 13
      java/Makefile
  20. 1
      java/rocksjni/write_batch.cc
  21. 16
      java/src/main/java/org/rocksdb/util/Environment.java
  22. 2
      java/src/test/java/org/rocksdb/RocksDBTest.java
  23. 19
      java/src/test/java/org/rocksdb/util/EnvironmentTest.java
  24. 2
      memtable/memtablerep_bench.cc
  25. 16
      options/options_helper.cc
  26. 16
      options/options_parser.cc
  27. 8
      port/port_posix.h
  28. 2
      port/stack_trace.cc
  29. 4
      table/cuckoo_table_builder_test.cc
  30. 6
      table/cuckoo_table_reader_test.cc
  31. 2
      table/partitioned_filter_block_test.cc
  32. 8
      table/plain_table_index.cc
  33. 2
      tools/auto_sanity_test.sh
  34. 2
      tools/rocksdb_dump_test.sh
  35. 2
      tools/run_flash_bench.sh
  36. 2
      tools/run_leveldb.sh
  37. 4
      util/bloom_test.cc
  38. 49
      util/coding.h
  39. 4
      util/delete_scheduler_test.cc
  40. 4
      util/testharness.h
  41. 8
      util/testutil.cc
  42. 14
      utilities/column_aware_encoding_test.cc
  43. 2
      utilities/persistent_cache/hash_table_test.cc
  44. 6
      utilities/persistent_cache/persistent_cache_bench.cc

@ -115,6 +115,15 @@ if(WIN32)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
endif() endif()
endif() endif()
option(WITH_MD_LIBRARY "build with MD" ON)
if(MSVC)
if(WITH_MD_LIBRARY)
set(RUNTIME_LIBRARY "MD")
else()
set(RUNTIME_LIBRARY "MT")
endif()
endif()
else() else()
option(WITH_SSE42 "build with SSE4.2" ON) option(WITH_SSE42 "build with SSE4.2" ON)
if(WITH_SSE42) if(WITH_SSE42)
@ -200,11 +209,11 @@ endif()
if(MSVC) if(MSVC)
if((${OPTIMIZE_DEBUG} EQUAL 1)) if((${OPTIMIZE_DEBUG} EQUAL 1))
message(STATUS "Debug optimization is enabled") message(STATUS "Debug optimization is enabled")
set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd") set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /${RUNTIME_LIBRARY}d")
else() else()
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /${RUNTIME_LIBRARY}d")
endif() endif()
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")

@ -106,3 +106,30 @@ makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SS
* **Windows**: * **Windows**:
* For building with MS Visual Studio 13 you will need Update 4 installed. * For building with MS Visual Studio 13 you will need Update 4 installed.
* Read and follow the instructions at CMakeLists.txt * Read and follow the instructions at CMakeLists.txt
* **AIX 6.1**
* Install AIX Toolbox rpms with gcc
* Use these environment variables:
export PORTABLE=1
export CC=gcc
export AR="ar -X64"
export EXTRA_ARFLAGS=-X64
export EXTRA_CFLAGS=-maix64
export EXTRA_CXXFLAGS=-maix64
export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
export LIBPATH=/opt/freeware/lib
export JAVA_HOME=/usr/java8_64
export PATH=/opt/freeware/bin:$PATH
* **Solaris Sparc**
* Install GCC 4.8.2 and higher.
* Use these environment variables:
export CC=gcc
export EXTRA_CFLAGS=-m64
export EXTRA_CXXFLAGS=-m64
export EXTRA_LDFLAGS=-m64
export PORTABLE=1
export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"

@ -14,7 +14,8 @@ CFLAGS += ${EXTRA_CFLAGS}
CXXFLAGS += ${EXTRA_CXXFLAGS} CXXFLAGS += ${EXTRA_CXXFLAGS}
LDFLAGS += $(EXTRA_LDFLAGS) LDFLAGS += $(EXTRA_LDFLAGS)
MACHINE ?= $(shell uname -m) MACHINE ?= $(shell uname -m)
ARFLAGS = rs ARFLAGS = ${EXTRA_ARFLAGS} rs
STRIPFLAGS = -S -x
# Transform parallel LOG output into something more readable. # Transform parallel LOG output into something more readable.
perl_command = perl -n \ perl_command = perl -n \
@ -154,7 +155,9 @@ missing_make_config_paths := $(shell \
$(foreach path, $(missing_make_config_paths), \ $(foreach path, $(missing_make_config_paths), \
$(warning Warning: $(path) dont exist)) $(warning Warning: $(path) dont exist))
ifneq ($(PLATFORM), IOS) ifeq ($(PLATFORM), OS_AIX)
# no debug info
else ifneq ($(PLATFORM), IOS)
CFLAGS += -g CFLAGS += -g
CXXFLAGS += -g CXXFLAGS += -g
else else
@ -162,6 +165,11 @@ else
OPT += -DNDEBUG OPT += -DNDEBUG
endif endif
ifeq ($(PLATFORM), OS_AIX)
ARFLAGS = -X64 rs
STRIPFLAGS = -X64 -x
endif
ifeq ($(PLATFORM), OS_SOLARIS) ifeq ($(PLATFORM), OS_SOLARIS)
PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99 PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99
endif endif
@ -194,6 +202,11 @@ ifdef COMPILE_WITH_TSAN
LUA_PATH = LUA_PATH =
endif endif
# AIX doesn't work with -pg
ifeq ($(PLATFORM), OS_AIX)
PROFILING_FLAGS =
endif
# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc. # USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
ifdef COMPILE_WITH_UBSAN ifdef COMPILE_WITH_UBSAN
DISABLE_JEMALLOC=1 DISABLE_JEMALLOC=1
@ -215,8 +228,14 @@ endif
export GTEST_THROW_ON_FAILURE=1 export GTEST_THROW_ON_FAILURE=1
export GTEST_HAS_EXCEPTIONS=1 export GTEST_HAS_EXCEPTIONS=1
GTEST_DIR = ./third-party/gtest-1.7.0/fused-src GTEST_DIR = ./third-party/gtest-1.7.0/fused-src
PLATFORM_CCFLAGS += -isystem $(GTEST_DIR) # AIX: pre-defined system headers are surrounded by an extern "C" block
PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR) ifeq ($(PLATFORM), OS_AIX)
PLATFORM_CCFLAGS += -I$(GTEST_DIR)
PLATFORM_CXXFLAGS += -I$(GTEST_DIR)
else
PLATFORM_CCFLAGS += -isystem $(GTEST_DIR)
PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR)
endif
# This (the first rule) must depend on "all". # This (the first rule) must depend on "all".
default: all default: all
@ -571,7 +590,8 @@ ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
# and create a randomly-named rocksdb.XXXX directory therein. # and create a randomly-named rocksdb.XXXX directory therein.
# We'll use that directory in the "make check" rules. # We'll use that directory in the "make check" rules.
ifeq ($(TMPD),) ifeq ($(TMPD),)
TMPD := $(shell f=/dev/shm; test -k $$f || f=/tmp; \ TMPDIR := $(shell echo $${TMPDIR:-/tmp})
TMPD := $(shell f=/dev/shm; test -k $$f || f=$(TMPDIR); \
perl -le 'use File::Temp "tempdir";' \ perl -le 'use File::Temp "tempdir";' \
-e 'print tempdir("'$$f'/rocksdb.XXXX", CLEANUP => 0)') -e 'print tempdir("'$$f'/rocksdb.XXXX", CLEANUP => 0)')
endif endif
@ -726,10 +746,12 @@ check: all
echo "===== Running $$t"; ./$$t || exit 1; done; \ echo "===== Running $$t"; ./$$t || exit 1; done; \
fi fi
rm -rf $(TMPD) rm -rf $(TMPD)
ifneq ($(PLATFORM), OS_AIX)
ifeq ($(filter -DROCKSDB_LITE,$(OPT)),) ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
python tools/ldb_test.py python tools/ldb_test.py
sh tools/rocksdb_dump_test.sh sh tools/rocksdb_dump_test.sh
endif endif
endif
# TODO add ldb_tests # TODO add ldb_tests
check_some: $(SUBSET) check_some: $(SUBSET)
@ -1402,14 +1424,18 @@ ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PA
ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar
SHA256_CMD = sha256sum SHA256_CMD = sha256sum
ZLIB_VER = 1.2.11 ZLIB_VER ?= 1.2.11
ZLIB_SHA256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 ZLIB_SHA256 ?= c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
BZIP2_VER = 1.0.6 ZLIB_DOWNLOAD_BASE ?= http://zlib.net
BZIP2_SHA256 = a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd BZIP2_VER ?= 1.0.6
SNAPPY_VER = 1.1.4 BZIP2_SHA256 ?= a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
SNAPPY_SHA256 = 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057 BZIP2_DOWNLOAD_BASE ?= http://www.bzip.org
LZ4_VER = 1.7.5 SNAPPY_VER ?= 1.1.4
LZ4_SHA256 = 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e SNAPPY_SHA256 ?= 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/releases/download
LZ4_VER ?= 1.7.5
LZ4_SHA256 ?= 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e
LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
ifeq ($(PLATFORM), OS_MACOSX) ifeq ($(PLATFORM), OS_MACOSX)
ROCKSDBJNILIB = librocksdbjni-osx.jnilib ROCKSDBJNILIB = librocksdbjni-osx.jnilib
@ -1432,47 +1458,53 @@ ifeq ($(PLATFORM), OS_SOLARIS)
JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris
SHA256_CMD = digest -a sha256 SHA256_CMD = digest -a sha256
endif endif
ifeq ($(PLATFORM), OS_AIX)
JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/aix
ROCKSDBJNILIB = librocksdbjni-aix.so
EXTRACT_SOURCES = gunzip < TAR_GZ | tar xvf -
SNAPPY_MAKE_TARGET = libsnappy.la
endif
libz.a: libz.a:
-rm -rf zlib-$(ZLIB_VER) -rm -rf zlib-$(ZLIB_VER)
curl -O -L http://zlib.net/zlib-$(ZLIB_VER).tar.gz curl -O -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz
ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \ ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \
if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \ if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \
echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \ echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \
exit 1; \ exit 1; \
fi fi
tar xvzf zlib-$(ZLIB_VER).tar.gz tar xvzf zlib-$(ZLIB_VER).tar.gz
cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC' ./configure --static && make cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && make
cp zlib-$(ZLIB_VER)/libz.a . cp zlib-$(ZLIB_VER)/libz.a .
libbz2.a: libbz2.a:
-rm -rf bzip2-$(BZIP2_VER) -rm -rf bzip2-$(BZIP2_VER)
curl -O -L http://www.bzip.org/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz curl -O -L ${BZIP2_DOWNLOAD_BASE}/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \ BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \
if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \ if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \
echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \ echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \
exit 1; \ exit 1; \
fi fi
tar xvzf bzip2-$(BZIP2_VER).tar.gz tar xvzf bzip2-$(BZIP2_VER).tar.gz
cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64' cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
cp bzip2-$(BZIP2_VER)/libbz2.a . cp bzip2-$(BZIP2_VER)/libbz2.a .
libsnappy.a: libsnappy.a:
-rm -rf snappy-$(SNAPPY_VER) -rm -rf snappy-$(SNAPPY_VER)
curl -O -L https://github.com/google/snappy/releases/download/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz curl -O -L ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \ SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \
if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \ if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \
echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \ echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \
exit 1; \ exit 1; \
fi fi
tar xvzf snappy-$(SNAPPY_VER).tar.gz tar xvzf snappy-$(SNAPPY_VER).tar.gz
cd snappy-$(SNAPPY_VER) && ./configure --with-pic --enable-static cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared
cd snappy-$(SNAPPY_VER) && make cd snappy-$(SNAPPY_VER) && make ${SNAPPY_MAKE_TARGET}
cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a . cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .
liblz4.a: liblz4.a:
-rm -rf lz4-$(LZ4_VER) -rm -rf lz4-$(LZ4_VER)
curl -O -L https://github.com/lz4/lz4/archive/v$(LZ4_VER).tar.gz curl -O -L ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz
LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \ LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \
if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \ if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \
@ -1480,7 +1512,7 @@ liblz4.a:
exit 1; \ exit 1; \
fi fi
tar xvzf lz4-$(LZ4_VER).tar.gz tar xvzf lz4-$(LZ4_VER).tar.gz
cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2' all cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
cp lz4-$(LZ4_VER)/lib/liblz4.a . cp lz4-$(LZ4_VER)/lib/liblz4.a .
# A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
@ -1504,7 +1536,7 @@ rocksdbjavastatic: $(java_static_libobjects)
-o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \ -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
$(java_static_libobjects) $(COVERAGEFLAGS) \ $(java_static_libobjects) $(COVERAGEFLAGS) \
$(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS) $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
cd java/target;strip -S -x $(ROCKSDBJNILIB) cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB)
cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB) cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class

@ -128,10 +128,17 @@ case "$TARGET_OS" in
;; ;;
SunOS) SunOS)
PLATFORM=OS_SOLARIS PLATFORM=OS_SOLARIS
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS" COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64"
# PORT_FILES=port/sunos/sunos_specific.cc # PORT_FILES=port/sunos/sunos_specific.cc
;; ;;
AIX)
PLATFORM=OS_AIX
CC=gcc
COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc"
# PORT_FILES=port/aix/aix_specific.cc
;;
FreeBSD) FreeBSD)
PLATFORM=OS_FREEBSD PLATFORM=OS_FREEBSD
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD" COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
@ -382,6 +389,18 @@ EOF
if [ "$?" = 0 ]; then if [ "$?" = 0 ]; then
PROFILING_FLAGS=-pg PROFILING_FLAGS=-pg
fi fi
# Test whether sync_file_range is supported for compatibility with an old glibc
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
#include <fcntl.h>
int main() {
int fd = open("/dev/null", 0);
sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
fi
fi fi
# TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning. # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
@ -427,7 +446,7 @@ elif test -z "$PORTABLE"; then
COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER " COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
COMMON_FLAGS="$COMMON_FLAGS -march=z10 " COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
else elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
COMMON_FLAGS="$COMMON_FLAGS -march=native " COMMON_FLAGS="$COMMON_FLAGS -march=native "
fi fi
fi fi

@ -320,8 +320,14 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) {
} // namespace rocksdb } // namespace rocksdb
int main(int argc, char** argv) { int main(int argc, char** argv) {
if (rocksdb::port::kLittleEndian) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
}
else {
fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n");
return 0;
}
} }
#else #else

@ -297,7 +297,7 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
std::vector<uint64_t> penalties; std::vector<uint64_t> penalties;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"DeleteScheduler::BackgroundEmptyTrash:Wait", "DeleteScheduler::BackgroundEmptyTrash:Wait",
[&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); }); [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"InstrumentedCondVar::TimedWaitInternal", [&](void* arg) { "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) {
// Turn timed wait into a simulated sleep // Turn timed wait into a simulated sleep

@ -1516,6 +1516,7 @@ class MockPersistentCache : public PersistentCache {
const size_t max_size_ = 10 * 1024; // 10KiB const size_t max_size_ = 10 * 1024; // 10KiB
}; };
#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented
TEST_F(DBTest2, PersistentCache) { TEST_F(DBTest2, PersistentCache) {
int num_iter = 80; int num_iter = 80;
@ -1579,6 +1580,7 @@ TEST_F(DBTest2, PersistentCache) {
} }
} }
} }
#endif // !OS_SOLARIS
namespace { namespace {
void CountSyncPoint() { void CountSyncPoint() {
@ -1702,6 +1704,7 @@ TEST_F(DBTest2, ReadAmpBitmap) {
options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES)); options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES));
} }
#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented
TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) { TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
if (dbname_.find("dev/shm") != std::string::npos) { if (dbname_.find("dev/shm") != std::string::npos) {
// /dev/shm dont support getting a unique file id, this mean that // /dev/shm dont support getting a unique file id, this mean that
@ -1786,6 +1789,7 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2, ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2,
total_loaded_bytes_iter1 + total_loaded_bytes_iter2); total_loaded_bytes_iter1 + total_loaded_bytes_iter2);
} }
#endif // !OS_SOLARIS
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) {

@ -7,7 +7,9 @@
#include "db/external_sst_file_ingestion_job.h" #include "db/external_sst_file_ingestion_job.h"
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS
#endif
#include <inttypes.h> #include <inttypes.h>
#include <algorithm> #include <algorithm>

@ -49,6 +49,7 @@ struct DBPropertyInfo {
extern const DBPropertyInfo* GetPropertyInfo(const Slice& property); extern const DBPropertyInfo* GetPropertyInfo(const Slice& property);
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#undef SCORE
enum class LevelStatType { enum class LevelStatType {
INVALID = 0, INVALID = 0,
NUM_FILES, NUM_FILES,

@ -141,7 +141,9 @@ class Counters {
// mapped to a levedb Put // mapped to a levedb Put
bool set(const std::string& key, uint64_t value) { bool set(const std::string& key, uint64_t value) {
// just treat the internal rep of int64 as the string // just treat the internal rep of int64 as the string
Slice slice((char *)&value, sizeof(value)); char buf[sizeof(value)];
EncodeFixed64(buf, value);
Slice slice(buf, sizeof(value));
auto s = db_->Put(put_option_, key, slice); auto s = db_->Put(put_option_, key, slice);
if (s.ok()) { if (s.ok()) {
@ -375,7 +377,9 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
Slice key("BatchSuccessiveMerge"); Slice key("BatchSuccessiveMerge");
uint64_t merge_value = 1; uint64_t merge_value = 1;
Slice merge_value_slice((char *)&merge_value, sizeof(merge_value)); char buf[sizeof(merge_value)];
EncodeFixed64(buf, merge_value);
Slice merge_value_slice(buf, sizeof(merge_value));
// Create the batch // Create the batch
WriteBatch batch; WriteBatch batch;

@ -633,18 +633,33 @@ TEST_F(PerfContextTest, MergeOperatorTime) {
SetPerfLevel(kEnableTime); SetPerfLevel(kEnableTime);
perf_context.Reset(); perf_context.Reset();
ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
#ifdef OS_SOLARIS
for (int i = 0; i < 100; i++) {
ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
}
#endif
EXPECT_GT(perf_context.merge_operator_time_nanos, 0); EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
ASSERT_OK(db->Flush(FlushOptions())); ASSERT_OK(db->Flush(FlushOptions()));
perf_context.Reset(); perf_context.Reset();
ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
#ifdef OS_SOLARIS
for (int i = 0; i < 100; i++) {
ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
}
#endif
EXPECT_GT(perf_context.merge_operator_time_nanos, 0); EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr));
perf_context.Reset(); perf_context.Reset();
ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
#ifdef OS_SOLARIS
for (int i = 0; i < 100; i++) {
ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
}
#endif
EXPECT_GT(perf_context.merge_operator_time_nanos, 0); EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
delete db; delete db;

@ -32,6 +32,7 @@ int main() {
#include "util/string_util.h" #include "util/string_util.h"
#include "util/testharness.h" #include "util/testharness.h"
#include "utilities/merge_operators.h" #include "utilities/merge_operators.h"
#include "util/coding.h"
using GFLAGS::ParseCommandLineFlags; using GFLAGS::ParseCommandLineFlags;
@ -65,12 +66,16 @@ struct TestKey {
}; };
// return a slice backed by test_key // return a slice backed by test_key
inline Slice TestKeyToSlice(const TestKey& test_key) { inline Slice TestKeyToSlice(std::string &s, const TestKey& test_key) {
return Slice((const char*)&test_key, sizeof(test_key)); s.clear();
PutFixed64(&s, test_key.prefix);
PutFixed64(&s, test_key.sorted);
return Slice(s.c_str(), s.size());
} }
inline const TestKey* SliceToTestKey(const Slice& slice) { inline const TestKey SliceToTestKey(const Slice& slice) {
return (const TestKey*)slice.data(); return TestKey(DecodeFixed64(slice.data()),
DecodeFixed64(slice.data() + 8));
} }
class TestKeyComparator : public Comparator { class TestKeyComparator : public Comparator {
@ -79,8 +84,10 @@ class TestKeyComparator : public Comparator {
// Compare needs to be aware of the possibility of a and/or b is // Compare needs to be aware of the possibility of a and/or b is
// prefix only // prefix only
virtual int Compare(const Slice& a, const Slice& b) const override { virtual int Compare(const Slice& a, const Slice& b) const override {
const TestKey* key_a = SliceToTestKey(a); const TestKey kkey_a = SliceToTestKey(a);
const TestKey* key_b = SliceToTestKey(b); const TestKey kkey_b = SliceToTestKey(b);
const TestKey *key_a = &kkey_a;
const TestKey *key_b = &kkey_b;
if (key_a->prefix != key_b->prefix) { if (key_a->prefix != key_b->prefix) {
if (key_a->prefix < key_b->prefix) return -1; if (key_a->prefix < key_b->prefix) return -1;
if (key_a->prefix > key_b->prefix) return 1; if (key_a->prefix > key_b->prefix) return 1;
@ -111,7 +118,8 @@ class TestKeyComparator : public Comparator {
} }
bool operator()(const TestKey& a, const TestKey& b) const { bool operator()(const TestKey& a, const TestKey& b) const {
return Compare(TestKeyToSlice(a), TestKeyToSlice(b)) < 0; std::string sa, sb;
return Compare(TestKeyToSlice(sa, a), TestKeyToSlice(sb, b)) < 0;
} }
virtual const char* Name() const override { virtual const char* Name() const override {
@ -128,30 +136,35 @@ namespace {
void PutKey(DB* db, WriteOptions write_options, uint64_t prefix, void PutKey(DB* db, WriteOptions write_options, uint64_t prefix,
uint64_t suffix, const Slice& value) { uint64_t suffix, const Slice& value) {
TestKey test_key(prefix, suffix); TestKey test_key(prefix, suffix);
Slice key = TestKeyToSlice(test_key); std::string s;
Slice key = TestKeyToSlice(s, test_key);
ASSERT_OK(db->Put(write_options, key, value)); ASSERT_OK(db->Put(write_options, key, value));
} }
void PutKey(DB* db, WriteOptions write_options, const TestKey& test_key, void PutKey(DB* db, WriteOptions write_options, const TestKey& test_key,
const Slice& value) { const Slice& value) {
Slice key = TestKeyToSlice(test_key); std::string s;
Slice key = TestKeyToSlice(s, test_key);
ASSERT_OK(db->Put(write_options, key, value)); ASSERT_OK(db->Put(write_options, key, value));
} }
void MergeKey(DB* db, WriteOptions write_options, const TestKey& test_key, void MergeKey(DB* db, WriteOptions write_options, const TestKey& test_key,
const Slice& value) { const Slice& value) {
Slice key = TestKeyToSlice(test_key); std::string s;
Slice key = TestKeyToSlice(s, test_key);
ASSERT_OK(db->Merge(write_options, key, value)); ASSERT_OK(db->Merge(write_options, key, value));
} }
void DeleteKey(DB* db, WriteOptions write_options, const TestKey& test_key) { void DeleteKey(DB* db, WriteOptions write_options, const TestKey& test_key) {
Slice key = TestKeyToSlice(test_key); std::string s;
Slice key = TestKeyToSlice(s, test_key);
ASSERT_OK(db->Delete(write_options, key)); ASSERT_OK(db->Delete(write_options, key));
} }
void SeekIterator(Iterator* iter, uint64_t prefix, uint64_t suffix) { void SeekIterator(Iterator* iter, uint64_t prefix, uint64_t suffix) {
TestKey test_key(prefix, suffix); TestKey test_key(prefix, suffix);
Slice key = TestKeyToSlice(test_key); std::string s;
Slice key = TestKeyToSlice(s, test_key);
iter->Seek(key); iter->Seek(key);
} }
@ -160,7 +173,8 @@ const std::string kNotFoundResult = "NOT_FOUND";
std::string Get(DB* db, const ReadOptions& read_options, uint64_t prefix, std::string Get(DB* db, const ReadOptions& read_options, uint64_t prefix,
uint64_t suffix) { uint64_t suffix) {
TestKey test_key(prefix, suffix); TestKey test_key(prefix, suffix);
Slice key = TestKeyToSlice(test_key); std::string s2;
Slice key = TestKeyToSlice(s2, test_key);
std::string result; std::string result;
Status s = db->Get(read_options, key, &result); Status s = db->Get(read_options, key, &result);
@ -527,7 +541,9 @@ TEST_F(PrefixTest, PrefixValid) {
PutKey(db.get(), write_options, 12345, 9, v19); PutKey(db.get(), write_options, 12345, 9, v19);
PutKey(db.get(), write_options, 12346, 8, v16); PutKey(db.get(), write_options, 12346, 8, v16);
db->Flush(FlushOptions()); db->Flush(FlushOptions());
db->Delete(write_options, TestKeyToSlice(TestKey(12346, 8))); TestKey test_key(12346, 8);
std::string s;
db->Delete(write_options, TestKeyToSlice(s, test_key));
db->Flush(FlushOptions()); db->Flush(FlushOptions());
read_options.prefix_same_as_start = true; read_options.prefix_same_as_start = true;
std::unique_ptr<Iterator> iter(db->NewIterator(read_options)); std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
@ -583,7 +599,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) { for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
TestKey test_key(prefix, sorted); TestKey test_key(prefix, sorted);
Slice key = TestKeyToSlice(test_key); std::string s;
Slice key = TestKeyToSlice(s, test_key);
std::string value(FLAGS_value_size, 0); std::string value(FLAGS_value_size, 0);
perf_context.Reset(); perf_context.Reset();
@ -605,7 +622,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
for (auto prefix : prefixes) { for (auto prefix : prefixes) {
TestKey test_key(prefix, FLAGS_items_per_prefix / 2); TestKey test_key(prefix, FLAGS_items_per_prefix / 2);
Slice key = TestKeyToSlice(test_key); std::string s;
Slice key = TestKeyToSlice(s, test_key);
std::string value = "v" + ToString(0); std::string value = "v" + ToString(0);
perf_context.Reset(); perf_context.Reset();
@ -639,7 +657,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) {
prefix < FLAGS_total_prefixes + 10000; prefix < FLAGS_total_prefixes + 10000;
prefix++) { prefix++) {
TestKey test_key(prefix, 0); TestKey test_key(prefix, 0);
Slice key = TestKeyToSlice(test_key); std::string s;
Slice key = TestKeyToSlice(s, test_key);
perf_context.Reset(); perf_context.Reset();
StopWatchNano timer(Env::Default(), true); StopWatchNano timer(Env::Default(), true);
@ -725,8 +744,8 @@ TEST_F(PrefixTest, PrefixSeekModePrev) {
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
if (FLAGS_enable_print) { if (FLAGS_enable_print) {
std::cout << "round " << prefix std::cout << "round " << prefix
<< " iter: " << SliceToTestKey(iter->key())->prefix << " iter: " << SliceToTestKey(iter->key()).prefix
<< SliceToTestKey(iter->key())->sorted << SliceToTestKey(iter->key()).sorted
<< " | map: " << it->first.prefix << it->first.sorted << " | " << " | map: " << it->first.prefix << it->first.sorted << " | "
<< iter->value().ToString() << " " << it->second << std::endl; << iter->value().ToString() << " " << it->second << std::endl;
} }
@ -747,16 +766,16 @@ TEST_F(PrefixTest, PrefixSeekModePrev) {
} }
} }
if (!iter->Valid() || if (!iter->Valid() ||
SliceToTestKey(iter->key())->prefix != stored_prefix) { SliceToTestKey(iter->key()).prefix != stored_prefix) {
break; break;
} }
stored_prefix = SliceToTestKey(iter->key())->prefix; stored_prefix = SliceToTestKey(iter->key()).prefix;
ASSERT_TRUE(iter->Valid()); ASSERT_TRUE(iter->Valid());
ASSERT_NE(it, whole_map.end()); ASSERT_NE(it, whole_map.end());
ASSERT_EQ(iter->value(), it->second); ASSERT_EQ(iter->value(), it->second);
if (FLAGS_enable_print) { if (FLAGS_enable_print) {
std::cout << "iter: " << SliceToTestKey(iter->key())->prefix std::cout << "iter: " << SliceToTestKey(iter->key()).prefix
<< SliceToTestKey(iter->key())->sorted << SliceToTestKey(iter->key()).sorted
<< " | map: " << it->first.prefix << it->first.sorted << " | map: " << it->first.prefix << it->first.sorted
<< " | " << iter->value().ToString() << " " << it->second << " | " << iter->value().ToString() << " " << it->second
<< std::endl; << std::endl;
@ -810,7 +829,8 @@ TEST_F(PrefixTest, PrefixSeekModePrev3) {
options.write_buffer_size = 1024 * 1024; options.write_buffer_size = 1024 * 1024;
std::string v14("v14"); std::string v14("v14");
TestKey upper_bound_key = TestKey(1, 5); TestKey upper_bound_key = TestKey(1, 5);
Slice upper_bound = TestKeyToSlice(upper_bound_key); std::string s;
Slice upper_bound = TestKeyToSlice(s, upper_bound_key);
{ {
DestroyDB(kDbName, Options()); DestroyDB(kDbName, Options());

14
env/env_chroot.cc vendored

@ -24,11 +24,18 @@ class ChrootEnv : public EnvWrapper {
public: public:
ChrootEnv(Env* base_env, const std::string& chroot_dir) ChrootEnv(Env* base_env, const std::string& chroot_dir)
: EnvWrapper(base_env) { : EnvWrapper(base_env) {
#if defined(OS_AIX)
char resolvedName[PATH_MAX];
char* real_chroot_dir = realpath(chroot_dir.c_str(), resolvedName);
#else
char* real_chroot_dir = realpath(chroot_dir.c_str(), nullptr); char* real_chroot_dir = realpath(chroot_dir.c_str(), nullptr);
#endif
// chroot_dir must exist so realpath() returns non-nullptr. // chroot_dir must exist so realpath() returns non-nullptr.
assert(real_chroot_dir != nullptr); assert(real_chroot_dir != nullptr);
chroot_dir_ = real_chroot_dir; chroot_dir_ = real_chroot_dir;
#if !defined(OS_AIX)
free(real_chroot_dir); free(real_chroot_dir);
#endif
} }
virtual Status NewSequentialFile(const std::string& fname, virtual Status NewSequentialFile(const std::string& fname,
@ -258,7 +265,12 @@ class ChrootEnv : public EnvWrapper {
} }
std::pair<Status, std::string> res; std::pair<Status, std::string> res;
res.second = chroot_dir_ + path; res.second = chroot_dir_ + path;
#if defined(OS_AIX)
char resolvedName[PATH_MAX];
char* normalized_path = realpath(res.second.c_str(), resolvedName);
#else
char* normalized_path = realpath(res.second.c_str(), nullptr); char* normalized_path = realpath(res.second.c_str(), nullptr);
#endif
if (normalized_path == nullptr) { if (normalized_path == nullptr) {
res.first = Status::NotFound(res.second, strerror(errno)); res.first = Status::NotFound(res.second, strerror(errno));
} else if (strlen(normalized_path) < chroot_dir_.size() || } else if (strlen(normalized_path) < chroot_dir_.size() ||
@ -269,7 +281,9 @@ class ChrootEnv : public EnvWrapper {
} else { } else {
res.first = Status::OK(); res.first = Status::OK();
} }
#if !defined(OS_AIX)
free(normalized_path); free(normalized_path);
#endif
return res; return res;
} }

28
env/env_posix.cc vendored

@ -159,7 +159,7 @@ class PosixEnv : public Env {
#ifdef ROCKSDB_LITE #ifdef ROCKSDB_LITE
return Status::IOError(fname, "Direct I/O not supported in RocksDB lite"); return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) #if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
flags |= O_DIRECT; flags |= O_DIRECT;
#endif #endif
} }
@ -206,7 +206,7 @@ class PosixEnv : public Env {
#ifdef ROCKSDB_LITE #ifdef ROCKSDB_LITE
return Status::IOError(fname, "Direct I/O not supported in RocksDB lite"); return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) #if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
flags |= O_DIRECT; flags |= O_DIRECT;
TEST_SYNC_POINT_CALLBACK("NewRandomAccessFile:O_DIRECT", &flags); TEST_SYNC_POINT_CALLBACK("NewRandomAccessFile:O_DIRECT", &flags);
#endif #endif
@ -271,7 +271,7 @@ class PosixEnv : public Env {
return Status::IOError(fname, "Direct I/O not supported in RocksDB lite"); return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
flags |= O_WRONLY; flags |= O_WRONLY;
#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) #if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
flags |= O_DIRECT; flags |= O_DIRECT;
#endif #endif
TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags); TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
@ -312,6 +312,14 @@ class PosixEnv : public Env {
s = IOError(fname, errno); s = IOError(fname, errno);
return s; return s;
} }
#elif defined(OS_SOLARIS)
if (directio(fd, DIRECTIO_ON) == -1) {
if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON
close(fd);
s = IOError(fname, errno);
return s;
}
}
#endif #endif
result->reset(new PosixWritableFile(fname, fd, options)); result->reset(new PosixWritableFile(fname, fd, options));
} else { } else {
@ -338,7 +346,7 @@ class PosixEnv : public Env {
return Status::IOError(fname, "Direct I/O not supported in RocksDB lite"); return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
flags |= O_WRONLY; flags |= O_WRONLY;
#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) #if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
flags |= O_DIRECT; flags |= O_DIRECT;
#endif #endif
TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags); TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
@ -385,6 +393,14 @@ class PosixEnv : public Env {
s = IOError(fname, errno); s = IOError(fname, errno);
return s; return s;
} }
#elif defined(OS_SOLARIS)
if (directio(fd, DIRECTIO_ON) == -1) {
if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON
close(fd);
s = IOError(fname, errno);
return s;
}
}
#endif #endif
result->reset(new PosixWritableFile(fname, fd, options)); result->reset(new PosixWritableFile(fname, fd, options));
} else { } else {
@ -668,10 +684,12 @@ class PosixEnv : public Env {
} }
virtual uint64_t NowNanos() override { virtual uint64_t NowNanos() override {
#if defined(OS_LINUX) || defined(OS_FREEBSD) #if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_AIX)
struct timespec ts; struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts); clock_gettime(CLOCK_MONOTONIC, &ts);
return static_cast<uint64_t>(ts.tv_sec) * 1000000000 + ts.tv_nsec; return static_cast<uint64_t>(ts.tv_sec) * 1000000000 + ts.tv_nsec;
#elif defined(OS_SOLARIS)
return gethrtime();
#elif defined(__MACH__) #elif defined(__MACH__)
clock_serv_t cclock; clock_serv_t cclock;
mach_timespec_t ts; mach_timespec_t ts;

10
env/env_test.cc vendored

@ -958,7 +958,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
// Create file. // Create file.
{ {
unique_ptr<WritableFile> wfile; unique_ptr<WritableFile> wfile;
#if !defined(OS_MACOSX) && !defined(OS_WIN) #if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
if (soptions.use_direct_writes) { if (soptions.use_direct_writes) {
soptions.use_direct_writes = false; soptions.use_direct_writes = false;
} }
@ -974,7 +974,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
unique_ptr<RandomAccessFile> file; unique_ptr<RandomAccessFile> file;
auto scratch = NewAligned(kSectorSize, 0); auto scratch = NewAligned(kSectorSize, 0);
Slice result; Slice result;
#if !defined(OS_MACOSX) && !defined(OS_WIN) #if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
if (soptions.use_direct_reads) { if (soptions.use_direct_reads) {
soptions.use_direct_reads = false; soptions.use_direct_reads = false;
} }
@ -991,7 +991,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
unique_ptr<SequentialFile> file; unique_ptr<SequentialFile> file;
auto scratch = NewAligned(kSectorSize, 0); auto scratch = NewAligned(kSectorSize, 0);
Slice result; Slice result;
#if !defined(OS_MACOSX) && !defined(OS_WIN) #if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
if (soptions.use_direct_reads) { if (soptions.use_direct_reads) {
soptions.use_direct_reads = false; soptions.use_direct_reads = false;
} }
@ -1136,7 +1136,7 @@ TEST_P(EnvPosixTestWithParam, Preallocation) {
unique_ptr<WritableFile> srcfile; unique_ptr<WritableFile> srcfile;
EnvOptions soptions; EnvOptions soptions;
soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; soptions.use_direct_reads = soptions.use_direct_writes = direct_io_;
#if !defined(OS_MACOSX) && !defined(OS_WIN) #if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
if (soptions.use_direct_writes) { if (soptions.use_direct_writes) {
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"NewWritableFile:O_DIRECT", [&](void* arg) { "NewWritableFile:O_DIRECT", [&](void* arg) {
@ -1198,7 +1198,7 @@ TEST_P(EnvPosixTestWithParam, ConsistentChildrenAttributes) {
oss << test::TmpDir(env_) << "/testfile_" << i; oss << test::TmpDir(env_) << "/testfile_" << i;
const std::string path = oss.str(); const std::string path = oss.str();
unique_ptr<WritableFile> file; unique_ptr<WritableFile> file;
#if !defined(OS_MACOSX) && !defined(OS_WIN) #if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX)
if (soptions.use_direct_writes) { if (soptions.use_direct_writes) {
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"NewWritableFile:O_DIRECT", [&](void* arg) { "NewWritableFile:O_DIRECT", [&](void* arg) {

10
env/io_posix.cc vendored

@ -267,7 +267,7 @@ size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) {
} }
#endif #endif
#if defined(OS_MACOSX) #if defined(OS_MACOSX) || defined(OS_AIX)
size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) { size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) {
if (max_size < kMaxVarint64Length * 3) { if (max_size < kMaxVarint64Length * 3) {
return 0; return 0;
@ -336,7 +336,7 @@ Status PosixRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result,
return s; return s;
} }
#if defined(OS_LINUX) || defined(OS_MACOSX) #if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX)
size_t PosixRandomAccessFile::GetUniqueId(char* id, size_t max_size) const { size_t PosixRandomAccessFile::GetUniqueId(char* id, size_t max_size) const {
return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size); return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size);
} }
@ -848,7 +848,7 @@ Status PosixWritableFile::Allocate(uint64_t offset, uint64_t len) {
} }
#endif #endif
#ifdef OS_LINUX #ifdef ROCKSDB_RANGESYNC_PRESENT
Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) { Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) {
assert(offset <= std::numeric_limits<off_t>::max()); assert(offset <= std::numeric_limits<off_t>::max());
assert(nbytes <= std::numeric_limits<off_t>::max()); assert(nbytes <= std::numeric_limits<off_t>::max());
@ -859,7 +859,9 @@ Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) {
return IOError(filename_, errno); return IOError(filename_, errno);
} }
} }
#endif
#ifdef OS_LINUX
size_t PosixWritableFile::GetUniqueId(char* id, size_t max_size) const { size_t PosixWritableFile::GetUniqueId(char* id, size_t max_size) const {
return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size); return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size);
} }
@ -961,9 +963,11 @@ Status PosixRandomRWFile::Close() {
PosixDirectory::~PosixDirectory() { close(fd_); } PosixDirectory::~PosixDirectory() { close(fd_); }
Status PosixDirectory::Fsync() { Status PosixDirectory::Fsync() {
#ifndef OS_AIX
if (fsync(fd_) == -1) { if (fsync(fd_) == -1) {
return IOError("directory", errno); return IOError("directory", errno);
} }
#endif
return Status::OK(); return Status::OK();
} }
} // namespace rocksdb } // namespace rocksdb

8
env/io_posix.h vendored

@ -15,7 +15,7 @@
// For non linux platform, the following macros are used only as place // For non linux platform, the following macros are used only as place
// holder. // holder.
#if !(defined OS_LINUX) && !(defined CYGWIN) #if !(defined OS_LINUX) && !(defined CYGWIN) && !(defined OS_AIX)
#define POSIX_FADV_NORMAL 0 /* [MC1] no further special treatment */ #define POSIX_FADV_NORMAL 0 /* [MC1] no further special treatment */
#define POSIX_FADV_RANDOM 1 /* [MC1] expect random page refs */ #define POSIX_FADV_RANDOM 1 /* [MC1] expect random page refs */
#define POSIX_FADV_SEQUENTIAL 2 /* [MC1] expect sequential page refs */ #define POSIX_FADV_SEQUENTIAL 2 /* [MC1] expect sequential page refs */
@ -79,7 +79,7 @@ class PosixRandomAccessFile : public RandomAccessFile {
virtual Status Read(uint64_t offset, size_t n, Slice* result, virtual Status Read(uint64_t offset, size_t n, Slice* result,
char* scratch) const override; char* scratch) const override;
#if defined(OS_LINUX) || defined(OS_MACOSX) #if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX)
virtual size_t GetUniqueId(char* id, size_t max_size) const override; virtual size_t GetUniqueId(char* id, size_t max_size) const override;
#endif #endif
virtual void Hint(AccessPattern pattern) override; virtual void Hint(AccessPattern pattern) override;
@ -126,8 +126,10 @@ class PosixWritableFile : public WritableFile {
#ifdef ROCKSDB_FALLOCATE_PRESENT #ifdef ROCKSDB_FALLOCATE_PRESENT
virtual Status Allocate(uint64_t offset, uint64_t len) override; virtual Status Allocate(uint64_t offset, uint64_t len) override;
#endif #endif
#ifdef OS_LINUX #ifdef ROCKSDB_RANGESYNC_PRESENT
virtual Status RangeSync(uint64_t offset, uint64_t nbytes) override; virtual Status RangeSync(uint64_t offset, uint64_t nbytes) override;
#endif
#ifdef OS_LINUX
virtual size_t GetUniqueId(char* id, size_t max_size) const override; virtual size_t GetUniqueId(char* id, size_t max_size) const override;
#endif #endif
}; };

@ -1,25 +1,31 @@
cmake_minimum_required(VERSION 2.6) cmake_minimum_required(VERSION 2.6)
set(JNI_NATIVE_SOURCES set(JNI_NATIVE_SOURCES
rocksjni/backupenginejni.cc
rocksjni/backupablejni.cc rocksjni/backupablejni.cc
rocksjni/backupenginejni.cc
rocksjni/checkpoint.cc rocksjni/checkpoint.cc
rocksjni/clock_cache.cc
rocksjni/columnfamilyhandle.cc rocksjni/columnfamilyhandle.cc
rocksjni/compaction_filter.cc rocksjni/compaction_filter.cc
rocksjni/compaction_options_fifo.cc
rocksjni/compaction_options_universal.cc
rocksjni/comparator.cc rocksjni/comparator.cc
rocksjni/comparatorjnicallback.cc rocksjni/comparatorjnicallback.cc
rocksjni/compression_options.cc
rocksjni/env.cc rocksjni/env.cc
rocksjni/env_options.cc rocksjni/env_options.cc
rocksjni/external_sst_file_info.cc rocksjni/external_sst_file_info.cc
rocksjni/filter.cc rocksjni/filter.cc
rocksjni/iterator.cc rocksjni/iterator.cc
rocksjni/loggerjnicallback.cc rocksjni/loggerjnicallback.cc
rocksjni/lru_cache.cc
rocksjni/memtablejni.cc rocksjni/memtablejni.cc
rocksjni/merge_operator.cc rocksjni/merge_operator.cc
rocksjni/options.cc rocksjni/options.cc
rocksjni/ratelimiterjni.cc rocksjni/ratelimiterjni.cc
rocksjni/remove_emptyvalue_compactionfilterjni.cc rocksjni/remove_emptyvalue_compactionfilterjni.cc
rocksjni/restorejni.cc rocksjni/restorejni.cc
rocksjni/rocksdb_exception_test.cc
rocksjni/rocksjni.cc rocksjni/rocksjni.cc
rocksjni/slice.cc rocksjni/slice.cc
rocksjni/snapshot.cc rocksjni/snapshot.cc
@ -29,38 +35,50 @@ set(JNI_NATIVE_SOURCES
rocksjni/transaction_log.cc rocksjni/transaction_log.cc
rocksjni/ttl.cc rocksjni/ttl.cc
rocksjni/write_batch.cc rocksjni/write_batch.cc
rocksjni/writebatchhandlerjnicallback.cc
rocksjni/write_batch_with_index.cc
rocksjni/write_batch_test.cc rocksjni/write_batch_test.cc
rocksjni/write_batch_with_index.cc
rocksjni/writebatchhandlerjnicallback.cc
) )
set(NATIVE_JAVA_CLASSES set(NATIVE_JAVA_CLASSES
org.rocksdb.AbstractCompactionFilter org.rocksdb.AbstractCompactionFilter
org.rocksdb.AbstractComparator org.rocksdb.AbstractComparator
org.rocksdb.AbstractImmutableNativeReference
org.rocksdb.AbstractNativeReference
org.rocksdb.AbstractRocksIterator
org.rocksdb.AbstractSlice org.rocksdb.AbstractSlice
org.rocksdb.BackupEngine org.rocksdb.AbstractWriteBatch
org.rocksdb.BackupableDB org.rocksdb.BackupableDB
org.rocksdb.BackupableDBOptions org.rocksdb.BackupableDBOptions
org.rocksdb.BackupEngine
org.rocksdb.BackupEngineTest
org.rocksdb.BlockBasedTableConfig org.rocksdb.BlockBasedTableConfig
org.rocksdb.BloomFilter org.rocksdb.BloomFilter
org.rocksdb.Cache
org.rocksdb.Checkpoint org.rocksdb.Checkpoint
org.rocksdb.ClockCache
org.rocksdb.ColumnFamilyHandle org.rocksdb.ColumnFamilyHandle
org.rocksdb.ColumnFamilyOptions org.rocksdb.ColumnFamilyOptions
org.rocksdb.CompactionOptionsFIFO
org.rocksdb.CompactionOptionsUniversal
org.rocksdb.Comparator org.rocksdb.Comparator
org.rocksdb.ComparatorOptions org.rocksdb.ComparatorOptions
org.rocksdb.CompressionOptions
org.rocksdb.DBOptions org.rocksdb.DBOptions
org.rocksdb.DirectComparator org.rocksdb.DirectComparator
org.rocksdb.DirectSlice org.rocksdb.DirectSlice
org.rocksdb.Env org.rocksdb.Env
org.rocksdb.EnvOptions org.rocksdb.EnvOptions
org.rocksdb.ExternalSstFileInfo org.rocksdb.ExternalSstFileInfo
org.rocksdb.FlushOptions
org.rocksdb.Filter org.rocksdb.Filter
org.rocksdb.GenericRateLimiterConfig org.rocksdb.FlushOptions
org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig
org.rocksdb.HashSkipListMemTableConfig org.rocksdb.HashSkipListMemTableConfig
org.rocksdb.Logger org.rocksdb.Logger
org.rocksdb.LRUCache
org.rocksdb.MemTableConfig
org.rocksdb.MergeOperator org.rocksdb.MergeOperator
org.rocksdb.NativeLibraryLoader
org.rocksdb.Options org.rocksdb.Options
org.rocksdb.PlainTableConfig org.rocksdb.PlainTableConfig
org.rocksdb.RateLimiter org.rocksdb.RateLimiter
@ -69,25 +87,31 @@ set(NATIVE_JAVA_CLASSES
org.rocksdb.RestoreBackupableDB org.rocksdb.RestoreBackupableDB
org.rocksdb.RestoreOptions org.rocksdb.RestoreOptions
org.rocksdb.RocksDB org.rocksdb.RocksDB
org.rocksdb.RocksDBExceptionTest
org.rocksdb.RocksEnv org.rocksdb.RocksEnv
org.rocksdb.RocksIterator org.rocksdb.RocksIterator
org.rocksdb.RocksIteratorInterface
org.rocksdb.RocksMemEnv org.rocksdb.RocksMemEnv
org.rocksdb.RocksMutableObject
org.rocksdb.RocksObject
org.rocksdb.SkipListMemTableConfig org.rocksdb.SkipListMemTableConfig
org.rocksdb.Slice org.rocksdb.Slice
org.rocksdb.Snapshot
org.rocksdb.SnapshotTest
org.rocksdb.SstFileWriter org.rocksdb.SstFileWriter
org.rocksdb.Statistics org.rocksdb.Statistics
org.rocksdb.StringAppendOperator
org.rocksdb.TableFormatConfig
org.rocksdb.TransactionLogIterator org.rocksdb.TransactionLogIterator
org.rocksdb.TtlDB org.rocksdb.TtlDB
org.rocksdb.VectorMemTableConfig org.rocksdb.VectorMemTableConfig
org.rocksdb.Snapshot org.rocksdb.WBWIRocksIterator
org.rocksdb.StringAppendOperator
org.rocksdb.WriteBatch org.rocksdb.WriteBatch
org.rocksdb.WriteBatch.Handler org.rocksdb.WriteBatch.Handler
org.rocksdb.WriteOptions
org.rocksdb.WriteBatchWithIndex
org.rocksdb.WBWIRocksIterator
org.rocksdb.WriteBatchTest org.rocksdb.WriteBatchTest
org.rocksdb.WriteBatchTestInternalHelper org.rocksdb.WriteBatchTestInternalHelper
org.rocksdb.WriteBatchWithIndex
org.rocksdb.WriteOptions
) )
include_directories($ENV{JAVA_HOME}/include) include_directories($ENV{JAVA_HOME}/include)
@ -111,9 +135,17 @@ if(NOT EXISTS ${JAVA_TEST_LIBDIR})
file(MAKE_DIRECTORY mkdir ${JAVA_TEST_LIBDIR}) file(MAKE_DIRECTORY mkdir ${JAVA_TEST_LIBDIR})
endif() endif()
if (DEFINED CUSTOM_REPO_URL)
set(SEARCH_REPO_URL ${CUSTOM_REPO_URL}/)
set(CENTRAL_REPO_URL ${CUSTOM_REPO_URL}/)
else ()
set(SEARCH_REPO_URL "http://search.maven.org/remotecontent?filepath=")
set(CENTRAL_REPO_URL "http://central.maven.org/maven2/")
endif()
if(NOT EXISTS ${JAVA_JUNIT_JAR}) if(NOT EXISTS ${JAVA_JUNIT_JAR})
message("Downloading ${JAVA_JUNIT_JAR}") message("Downloading ${JAVA_JUNIT_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) file(DOWNLOAD ${SEARCH_REPO_URL}junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code) list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0) if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}") message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}")
@ -122,7 +154,7 @@ if(NOT EXISTS ${JAVA_JUNIT_JAR})
endif() endif()
if(NOT EXISTS ${JAVA_HAMCR_JAR}) if(NOT EXISTS ${JAVA_HAMCR_JAR})
message("Downloading ${JAVA_HAMCR_JAR}") message("Downloading ${JAVA_HAMCR_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) file(DOWNLOAD ${SEARCH_REPO_URL}org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code) list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0) if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}") message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}")
@ -131,7 +163,7 @@ if(NOT EXISTS ${JAVA_HAMCR_JAR})
endif() endif()
if(NOT EXISTS ${JAVA_MOCKITO_JAR}) if(NOT EXISTS ${JAVA_MOCKITO_JAR})
message("Downloading ${JAVA_MOCKITO_JAR}") message("Downloading ${JAVA_MOCKITO_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) file(DOWNLOAD ${SEARCH_REPO_URL}org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code) list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0) if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}") message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}")
@ -140,7 +172,7 @@ if(NOT EXISTS ${JAVA_MOCKITO_JAR})
endif() endif()
if(NOT EXISTS ${JAVA_CGLIB_JAR}) if(NOT EXISTS ${JAVA_CGLIB_JAR})
message("Downloading ${JAVA_CGLIB_JAR}") message("Downloading ${JAVA_CGLIB_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) file(DOWNLOAD ${SEARCH_REPO_URL}cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code) list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0) if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}") message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}")
@ -149,7 +181,7 @@ if(NOT EXISTS ${JAVA_CGLIB_JAR})
endif() endif()
if(NOT EXISTS ${JAVA_ASSERTJ_JAR}) if(NOT EXISTS ${JAVA_ASSERTJ_JAR})
message("Downloading ${JAVA_ASSERTJ_JAR}") message("Downloading ${JAVA_ASSERTJ_JAR}")
file(DOWNLOAD http://central.maven.org/maven2/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) file(DOWNLOAD ${CENTRAL_REPO_URL}org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code) list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0) if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}") message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}")

@ -157,6 +157,9 @@ ifneq ($(DEBUG_LEVEL),0)
JAVA_ARGS = -ea -Xcheck:jni JAVA_ARGS = -ea -Xcheck:jni
endif endif
SEARCH_REPO_URL?=http://search.maven.org/remotecontent?filepath=
CENTRAL_REPO_URL?=http://central.maven.org/maven2/
clean: clean:
$(AM_V_at)rm -rf include/* $(AM_V_at)rm -rf include/*
$(AM_V_at)rm -rf test-libs/ $(AM_V_at)rm -rf test-libs/
@ -198,11 +201,11 @@ column_family_sample: java
resolve_test_deps: resolve_test_deps:
test -d "$(JAVA_TEST_LIBDIR)" || mkdir -p "$(JAVA_TEST_LIBDIR)" test -d "$(JAVA_TEST_LIBDIR)" || mkdir -p "$(JAVA_TEST_LIBDIR)"
test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) http://search.maven.org/remotecontent?filepath=junit/junit/4.12/junit-4.12.jar test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) $(SEARCH_REPO_URL)junit/junit/4.12/junit-4.12.jar
test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) $(SEARCH_REPO_URL)org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar
test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" http://search.maven.org/remotecontent?filepath=org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" $(SEARCH_REPO_URL)org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar
test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" http://search.maven.org/remotecontent?filepath=cglib/cglib/2.2.2/cglib-2.2.2.jar test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" $(SEARCH_REPO_URL)cglib/cglib/2.2.2/cglib-2.2.2.jar
test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" http://central.maven.org/maven2/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" $(CENTRAL_REPO_URL)org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar
java_test: java resolve_test_deps java_test: java resolve_test_deps
$(AM_V_GEN)mkdir -p $(TEST_CLASSES) $(AM_V_GEN)mkdir -p $(TEST_CLASSES)

@ -21,7 +21,6 @@
#include "rocksjni/writebatchhandlerjnicallback.h" #include "rocksjni/writebatchhandlerjnicallback.h"
#include "table/scoped_arena_iterator.h" #include "table/scoped_arena_iterator.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/testharness.h"
/* /*
* Class: org_rocksdb_WriteBatch * Class: org_rocksdb_WriteBatch

@ -16,10 +16,13 @@ public class Environment {
return (OS.contains("mac")); return (OS.contains("mac"));
} }
public static boolean isAix() {
return OS.contains("aix");
}
public static boolean isUnix() { public static boolean isUnix() {
return (OS.contains("nix") || return OS.contains("nix") ||
OS.contains("nux") || OS.contains("nux");
OS.contains("aix"));
} }
public static boolean isSolaris() { public static boolean isSolaris() {
@ -27,6 +30,9 @@ public class Environment {
} }
public static boolean is64Bit() { public static boolean is64Bit() {
if (ARCH.indexOf("sparcv9") >= 0) {
return true;
}
return (ARCH.indexOf("64") > 0); return (ARCH.indexOf("64") > 0);
} }
@ -48,6 +54,8 @@ public class Environment {
} }
} else if (isMac()) { } else if (isMac()) {
return String.format("%sjni-osx", name); return String.format("%sjni-osx", name);
} else if (isAix() && is64Bit()) {
return String.format("%sjni-aix64", name);
} else if (isSolaris()) { } else if (isSolaris()) {
final String arch = is64Bit() ? "64" : "32"; final String arch = is64Bit() ? "64" : "32";
return String.format("%sjni-solaris%s", name, arch); return String.format("%sjni-solaris%s", name, arch);
@ -63,7 +71,7 @@ public class Environment {
} }
private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) { private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) {
if (isUnix() || isSolaris()) { if (isUnix() || isAix() || isSolaris()) {
return libraryFileName + ".so"; return libraryFileName + ".so";
} else if (isMac()) { } else if (isMac()) {
return libraryFileName + (shared ? ".dylib" : ".jnilib"); return libraryFileName + (shared ? ".dylib" : ".jnilib");

@ -53,7 +53,7 @@ public class RocksDBTest {
} catch (final RocksDBException e) { } catch (final RocksDBException e) {
assertThat(e.getStatus().getCode()).isEqualTo(Status.Code.IOError); assertThat(e.getStatus().getCode()).isEqualTo(Status.Code.IOError);
assertThat(e.getStatus().getSubCode()).isEqualTo(Status.SubCode.None); assertThat(e.getStatus().getSubCode()).isEqualTo(Status.SubCode.None);
assertThat(e.getStatus().getState()).startsWith("lock "); assertThat(e.getStatus().getState()).contains("lock ");
} }
} }
} }

@ -1,4 +1,4 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. // Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the // This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant // LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory. // of patent rights can be found in the PATENTS file in the same directory.
@ -70,15 +70,16 @@ public class EnvironmentTest {
isEqualTo("librocksdbjni-linux32.so"); isEqualTo("librocksdbjni-linux32.so");
assertThat(Environment.getSharedLibraryFileName("rocksdb")). assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so"); isEqualTo("librocksdbjni.so");
}
@Test(expected = UnsupportedOperationException.class)
public void aix32() {
// AIX // AIX
setEnvironmentClassFields("aix", "32"); setEnvironmentClassFields("aix", "32");
assertThat(Environment.isWindows()).isFalse(); assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()). assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".so"); isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")). Environment.getJniLibraryFileName("rocksdb");
isEqualTo("librocksdbjni-linux32.so");
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
} }
@Test @Test
@ -106,7 +107,7 @@ public class EnvironmentTest {
assertThat(Environment.getJniLibraryExtension()). assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".so"); isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")). assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-linux64.so"); isEqualTo("librocksdbjni-aix64.so");
assertThat(Environment.getSharedLibraryFileName("rocksdb")). assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so"); isEqualTo("librocksdbjni.so");
} }
@ -129,12 +130,6 @@ public class EnvironmentTest {
isEqualTo("librocksdbjni.dll"); isEqualTo("librocksdbjni.dll");
} }
@Test(expected = UnsupportedOperationException.class)
public void win32(){
setEnvironmentClassFields("win", "32");
Environment.getJniLibraryFileName("rocksdb");
}
private void setEnvironmentClassFields(String osName, private void setEnvironmentClassFields(String osName,
String osArch) { String osArch) {
setEnvironmentClassField(OS_FIELD_NAME, osName); setEnvironmentClassField(OS_FIELD_NAME, osName);

@ -7,7 +7,9 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS
#endif
#ifndef GFLAGS #ifndef GFLAGS
#include <cstdio> #include <cstdio>

@ -320,10 +320,10 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
*reinterpret_cast<uint32_t*>(opt_address) = ParseUint32(value); *reinterpret_cast<uint32_t*>(opt_address) = ParseUint32(value);
break; break;
case OptionType::kUInt64T: case OptionType::kUInt64T:
*reinterpret_cast<uint64_t*>(opt_address) = ParseUint64(value); PutUnaligned(reinterpret_cast<uint64_t*>(opt_address), ParseUint64(value));
break; break;
case OptionType::kSizeT: case OptionType::kSizeT:
*reinterpret_cast<size_t*>(opt_address) = ParseSizeT(value); PutUnaligned(reinterpret_cast<size_t*>(opt_address), ParseSizeT(value));
break; break;
case OptionType::kString: case OptionType::kString:
*reinterpret_cast<std::string*>(opt_address) = value; *reinterpret_cast<std::string*>(opt_address) = value;
@ -404,10 +404,18 @@ bool SerializeSingleOptionHelper(const char* opt_address,
*value = ToString(*(reinterpret_cast<const uint32_t*>(opt_address))); *value = ToString(*(reinterpret_cast<const uint32_t*>(opt_address)));
break; break;
case OptionType::kUInt64T: case OptionType::kUInt64T:
*value = ToString(*(reinterpret_cast<const uint64_t*>(opt_address))); {
uint64_t v;
GetUnaligned(reinterpret_cast<const uint64_t*>(opt_address), &v);
*value = ToString(v);
}
break; break;
case OptionType::kSizeT: case OptionType::kSizeT:
*value = ToString(*(reinterpret_cast<const size_t*>(opt_address))); {
size_t v;
GetUnaligned(reinterpret_cast<const size_t*>(opt_address), &v);
*value = ToString(v);
}
break; break;
case OptionType::kDouble: case OptionType::kDouble:
*value = ToString(*(reinterpret_cast<const double*>(opt_address))); *value = ToString(*(reinterpret_cast<const double*>(opt_address)));

@ -528,11 +528,19 @@ bool AreEqualOptions(
return (*reinterpret_cast<const uint32_t*>(offset1) == return (*reinterpret_cast<const uint32_t*>(offset1) ==
*reinterpret_cast<const uint32_t*>(offset2)); *reinterpret_cast<const uint32_t*>(offset2));
case OptionType::kUInt64T: case OptionType::kUInt64T:
return (*reinterpret_cast<const uint64_t*>(offset1) == {
*reinterpret_cast<const uint64_t*>(offset2)); uint64_t v1, v2;
GetUnaligned(reinterpret_cast<const uint64_t*>(offset1), &v1);
GetUnaligned(reinterpret_cast<const uint64_t*>(offset2), &v2);
return (v1 == v2);
}
case OptionType::kSizeT: case OptionType::kSizeT:
return (*reinterpret_cast<const size_t*>(offset1) == {
*reinterpret_cast<const size_t*>(offset2)); size_t v1, v2;
GetUnaligned(reinterpret_cast<const size_t*>(offset1), &v1);
GetUnaligned(reinterpret_cast<const size_t*>(offset2), &v2);
return (v1 == v2);
}
case OptionType::kString: case OptionType::kString:
return (*reinterpret_cast<const std::string*>(offset1) == return (*reinterpret_cast<const std::string*>(offset1) ==
*reinterpret_cast<const std::string*>(offset2)); *reinterpret_cast<const std::string*>(offset2));

@ -35,6 +35,12 @@
#else #else
#define PLATFORM_IS_LITTLE_ENDIAN false #define PLATFORM_IS_LITTLE_ENDIAN false
#endif #endif
#include <alloca.h>
#elif defined(OS_AIX)
#include <sys/types.h>
#include <arpa/nameser_compat.h>
#define PLATFORM_IS_LITTLE_ENDIAN (BYTE_ORDER == LITTLE_ENDIAN)
#include <alloca.h>
#elif defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || \ #elif defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || \
defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID) defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID)
#include <sys/endian.h> #include <sys/endian.h>
@ -56,7 +62,7 @@
#if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\
defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\ defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\
defined(OS_ANDROID) || defined(CYGWIN) defined(OS_ANDROID) || defined(CYGWIN) || defined(OS_AIX)
// Use fread/fwrite/fflush on platforms without _unlocked variants // Use fread/fwrite/fflush on platforms without _unlocked variants
#define fread_unlocked fread #define fread_unlocked fread
#define fwrite_unlocked fwrite #define fwrite_unlocked fwrite

@ -6,7 +6,7 @@
#include "port/stack_trace.h" #include "port/stack_trace.h"
#if defined(ROCKSDB_LITE) || !(defined(ROCKSDB_BACKTRACE) || defined(OS_MACOSX)) || \ #if defined(ROCKSDB_LITE) || !(defined(ROCKSDB_BACKTRACE) || defined(OS_MACOSX)) || \
defined(CYGWIN) || defined(OS_FREEBSD) defined(CYGWIN) || defined(OS_FREEBSD) || defined(OS_SOLARIS)
// noop // noop

@ -66,8 +66,8 @@ class CuckooBuilderTest : public testing::Test {
ASSERT_EQ(expected_unused_bucket.substr(0, ASSERT_EQ(expected_unused_bucket.substr(0,
props->fixed_key_len), unused_key); props->fixed_key_len), unused_key);
uint32_t value_len_found = uint64_t value_len_found =
*reinterpret_cast<const uint32_t*>(props->user_collected_properties[ *reinterpret_cast<const uint64_t*>(props->user_collected_properties[
CuckooTablePropertyNames::kValueLength].data()); CuckooTablePropertyNames::kValueLength].data());
ASSERT_EQ(values.empty() ? 0 : values[0].size(), value_len_found); ASSERT_EQ(values.empty() ? 0 : values[0].size(), value_len_found);
ASSERT_EQ(props->raw_value_size, values.size()*value_len_found); ASSERT_EQ(props->raw_value_size, values.size()*value_len_found);

@ -544,9 +544,15 @@ TEST_F(CuckooReaderTest, TestReadPerformance) {
} // namespace rocksdb } // namespace rocksdb
int main(int argc, char** argv) { int main(int argc, char** argv) {
if (rocksdb::port::kLittleEndian) {
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
ParseCommandLineFlags(&argc, &argv, true); ParseCommandLineFlags(&argc, &argv, true);
return RUN_ALL_TESTS(); return RUN_ALL_TESTS();
}
else {
fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n");
return 0;
}
} }
#endif // GFLAGS. #endif // GFLAGS.

@ -56,7 +56,7 @@ class PartitionedFilterBlockTest : public testing::Test {
int num_keys = sizeof(keys) / sizeof(*keys); int num_keys = sizeof(keys) / sizeof(*keys);
uint64_t max_key_size = 0; uint64_t max_key_size = 0;
for (int i = 1; i < num_keys; i++) { for (int i = 1; i < num_keys; i++) {
max_key_size = std::max(max_key_size, keys[i].size()); max_key_size = std::max(max_key_size, static_cast<uint64_t>(keys[i].size()));
} }
uint64_t max_index_size = num_keys * (max_key_size + 8 /*handle*/); uint64_t max_index_size = num_keys * (max_key_size + 8 /*handle*/);
return max_index_size; return max_index_size;

@ -44,7 +44,7 @@ Status PlainTableIndex::InitFromRawData(Slice data) {
PlainTableIndex::IndexSearchResult PlainTableIndex::GetOffset( PlainTableIndex::IndexSearchResult PlainTableIndex::GetOffset(
uint32_t prefix_hash, uint32_t* bucket_value) const { uint32_t prefix_hash, uint32_t* bucket_value) const {
int bucket = GetBucketIdFromHash(prefix_hash, index_size_); int bucket = GetBucketIdFromHash(prefix_hash, index_size_);
*bucket_value = index_[bucket]; GetUnaligned(index_ + bucket, bucket_value);
if ((*bucket_value & kSubIndexMask) == kSubIndexMask) { if ((*bucket_value & kSubIndexMask) == kSubIndexMask) {
*bucket_value ^= kSubIndexMask; *bucket_value ^= kSubIndexMask;
return kSubindex; return kSubindex;
@ -175,15 +175,15 @@ Slice PlainTableIndexBuilder::FillIndexes(
switch (num_keys_for_bucket) { switch (num_keys_for_bucket) {
case 0: case 0:
// No key for bucket // No key for bucket
index[i] = PlainTableIndex::kMaxFileSize; PutUnaligned(index + i, (uint32_t)PlainTableIndex::kMaxFileSize);
break; break;
case 1: case 1:
// point directly to the file offset // point directly to the file offset
index[i] = hash_to_offsets[i]->offset; PutUnaligned(index + i, hash_to_offsets[i]->offset);
break; break;
default: default:
// point to second level indexes. // point to second level indexes.
index[i] = sub_index_offset | PlainTableIndex::kSubIndexMask; PutUnaligned(index + i, sub_index_offset | PlainTableIndex::kSubIndexMask);
char* prev_ptr = &sub_index[sub_index_offset]; char* prev_ptr = &sub_index[sub_index_offset];
char* cur_ptr = EncodeVarint32(prev_ptr, num_keys_for_bucket); char* cur_ptr = EncodeVarint32(prev_ptr, num_keys_for_bucket);
sub_index_offset += static_cast<uint32_t>(cur_ptr - prev_ptr); sub_index_offset += static_cast<uint32_t>(cur_ptr - prev_ptr);

@ -1,4 +1,4 @@
TMP_DIR="/tmp/rocksdb-sanity-test" TMP_DIR="${TMPDIR:-/tmp}/rocksdb-sanity-test"
if [ "$#" -lt 2 ]; then if [ "$#" -lt 2 ]; then
echo "usage: ./auto_sanity_test.sh [new_commit] [old_commit]" echo "usage: ./auto_sanity_test.sh [new_commit] [old_commit]"

@ -1,4 +1,4 @@
TESTDIR=`mktemp -d /tmp/rocksdb-dump-test.XXXXX` TESTDIR=`mktemp -d ${TMPDIR:-/tmp}/rocksdb-dump-test.XXXXX`
DUMPFILE="tools/sample-dump.dmp" DUMPFILE="tools/sample-dump.dmp"
# Verify that the sample dump file is undumpable and then redumpable. # Verify that the sample dump file is undumpable and then redumpable.

@ -102,7 +102,7 @@ if [[ $skip_low_pri_tests == 1 ]]; then
echo "Skipping some non-critical tests because SKIP_LOW_PRI_TESTS is set." echo "Skipping some non-critical tests because SKIP_LOW_PRI_TESTS is set."
fi fi
output_dir="/tmp/output" output_dir="${TMPDIR:-/tmp}/output"
ARGS="\ ARGS="\
OUTPUT_DIR=$output_dir \ OUTPUT_DIR=$output_dir \

@ -71,7 +71,7 @@ db_dir=${DATA_DIR:-"/tmp/rocksdb/"}
do_setup=${DO_SETUP:-1} do_setup=${DO_SETUP:-1}
save_setup=${SAVE_SETUP:-0} save_setup=${SAVE_SETUP:-0}
output_dir="/tmp/output" output_dir="${TMPDIR:-/tmp}/output"
ARGS="\ ARGS="\
OUTPUT_DIR=$output_dir \ OUTPUT_DIR=$output_dir \

@ -33,7 +33,9 @@ namespace rocksdb {
static const int kVerbose = 1; static const int kVerbose = 1;
static Slice Key(int i, char* buffer) { static Slice Key(int i, char* buffer) {
memcpy(buffer, &i, sizeof(i)); std::string s;
PutFixed32(&s, static_cast<uint32_t>(i));
memcpy(buffer, s.c_str(), sizeof(i));
return Slice(buffer, sizeof(i)); return Slice(buffer, sizeof(i));
} }

@ -21,6 +21,11 @@
#include "rocksdb/write_batch.h" #include "rocksdb/write_batch.h"
#include "port/port.h" #include "port/port.h"
// Some processors does not allow unaligned access to memory
#if defined(__sparc)
#define PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED
#endif
namespace rocksdb { namespace rocksdb {
// The maximum length of a varint in bytes for 64-bit. // The maximum length of a varint in bytes for 64-bit.
@ -127,20 +132,20 @@ inline const char* GetVarint32Ptr(const char* p,
// -- Implementation of the functions declared above // -- Implementation of the functions declared above
inline void EncodeFixed32(char* buf, uint32_t value) { inline void EncodeFixed32(char* buf, uint32_t value) {
#if __BYTE_ORDER == __LITTLE_ENDIAN if (port::kLittleEndian) {
memcpy(buf, &value, sizeof(value)); memcpy(buf, &value, sizeof(value));
#else } else {
buf[0] = value & 0xff; buf[0] = value & 0xff;
buf[1] = (value >> 8) & 0xff; buf[1] = (value >> 8) & 0xff;
buf[2] = (value >> 16) & 0xff; buf[2] = (value >> 16) & 0xff;
buf[3] = (value >> 24) & 0xff; buf[3] = (value >> 24) & 0xff;
#endif }
} }
inline void EncodeFixed64(char* buf, uint64_t value) { inline void EncodeFixed64(char* buf, uint64_t value) {
#if __BYTE_ORDER == __LITTLE_ENDIAN if (port::kLittleEndian) {
memcpy(buf, &value, sizeof(value)); memcpy(buf, &value, sizeof(value));
#else } else {
buf[0] = value & 0xff; buf[0] = value & 0xff;
buf[1] = (value >> 8) & 0xff; buf[1] = (value >> 8) & 0xff;
buf[2] = (value >> 16) & 0xff; buf[2] = (value >> 16) & 0xff;
@ -149,30 +154,30 @@ inline void EncodeFixed64(char* buf, uint64_t value) {
buf[5] = (value >> 40) & 0xff; buf[5] = (value >> 40) & 0xff;
buf[6] = (value >> 48) & 0xff; buf[6] = (value >> 48) & 0xff;
buf[7] = (value >> 56) & 0xff; buf[7] = (value >> 56) & 0xff;
#endif }
} }
// Pull the last 8 bits and cast it to a character // Pull the last 8 bits and cast it to a character
inline void PutFixed32(std::string* dst, uint32_t value) { inline void PutFixed32(std::string* dst, uint32_t value) {
#if __BYTE_ORDER__ == __LITTLE_ENDIAN__ if (port::kLittleEndian) {
dst->append(const_cast<const char*>(reinterpret_cast<char*>(&value)), dst->append(const_cast<const char*>(reinterpret_cast<char*>(&value)),
sizeof(value)); sizeof(value));
#else } else {
char buf[sizeof(value)]; char buf[sizeof(value)];
EncodeFixed32(buf, value); EncodeFixed32(buf, value);
dst->append(buf, sizeof(buf)); dst->append(buf, sizeof(buf));
#endif }
} }
inline void PutFixed64(std::string* dst, uint64_t value) { inline void PutFixed64(std::string* dst, uint64_t value) {
#if __BYTE_ORDER__ == __LITTLE_ENDIAN__ if (port::kLittleEndian) {
dst->append(const_cast<const char*>(reinterpret_cast<char*>(&value)), dst->append(const_cast<const char*>(reinterpret_cast<char*>(&value)),
sizeof(value)); sizeof(value));
#else } else {
char buf[sizeof(value)]; char buf[sizeof(value)];
EncodeFixed64(buf, value); EncodeFixed64(buf, value);
dst->append(buf, sizeof(buf)); dst->append(buf, sizeof(buf));
#endif }
} }
inline void PutVarint32(std::string* dst, uint32_t v) { inline void PutVarint32(std::string* dst, uint32_t v) {
@ -346,4 +351,24 @@ inline Slice GetSliceUntil(Slice* slice, char delimiter) {
return ret; return ret;
} }
template<class T>
inline void PutUnaligned(T *memory, const T &value) {
#if defined(PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED)
char *nonAlignedMemory = reinterpret_cast<char*>(memory);
memcpy(nonAlignedMemory, reinterpret_cast<const char*>(&value), sizeof(T));
#else
*memory = value;
#endif
}
template<class T>
inline void GetUnaligned(const T *memory, T *value) {
#if defined(PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED)
char *nonAlignedMemory = reinterpret_cast<char*>(value);
memcpy(nonAlignedMemory, reinterpret_cast<const char*>(memory), sizeof(T));
#else
*value = *memory;
#endif
}
} // namespace rocksdb } // namespace rocksdb

@ -91,7 +91,7 @@ TEST_F(DeleteSchedulerTest, BasicRateLimiting) {
std::vector<uint64_t> penalties; std::vector<uint64_t> penalties;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"DeleteScheduler::BackgroundEmptyTrash:Wait", "DeleteScheduler::BackgroundEmptyTrash:Wait",
[&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); }); [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
int num_files = 100; // 100 files int num_files = 100; // 100 files
uint64_t file_size = 1024; // every file is 1 kb uint64_t file_size = 1024; // every file is 1 kb
@ -158,7 +158,7 @@ TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) {
std::vector<uint64_t> penalties; std::vector<uint64_t> penalties;
rocksdb::SyncPoint::GetInstance()->SetCallBack( rocksdb::SyncPoint::GetInstance()->SetCallBack(
"DeleteScheduler::BackgroundEmptyTrash:Wait", "DeleteScheduler::BackgroundEmptyTrash:Wait",
[&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); }); [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
int thread_cnt = 10; int thread_cnt = 10;
int num_files = 10; // 10 files per thread int num_files = 10; // 10 files per thread

@ -9,7 +9,11 @@
#pragma once #pragma once
#ifdef OS_AIX
#include "gtest/gtest.h"
#else
#include <gtest/gtest.h> #include <gtest/gtest.h>
#endif
#include <string> #include <string>
#include "rocksdb/env.h" #include "rocksdb/env.h"

@ -94,9 +94,13 @@ class Uint64ComparatorImpl : public Comparator {
assert(a.size() == sizeof(uint64_t) && b.size() == sizeof(uint64_t)); assert(a.size() == sizeof(uint64_t) && b.size() == sizeof(uint64_t));
const uint64_t* left = reinterpret_cast<const uint64_t*>(a.data()); const uint64_t* left = reinterpret_cast<const uint64_t*>(a.data());
const uint64_t* right = reinterpret_cast<const uint64_t*>(b.data()); const uint64_t* right = reinterpret_cast<const uint64_t*>(b.data());
if (*left == *right) { uint64_t leftValue;
uint64_t rightValue;
GetUnaligned(left, &leftValue);
GetUnaligned(right, &rightValue);
if (leftValue == rightValue) {
return 0; return 0;
} else if (*left < *right) { } else if (leftValue < rightValue) {
return -1; return -1;
} else { } else {
return 1; return 1;

@ -59,9 +59,9 @@ TEST_P(ColumnAwareEncodingTestWithSize, NoCompressionEncodeDecode) {
const char* encoded_data_ptr = encoded_data.c_str(); const char* encoded_data_ptr = encoded_data.c_str();
uint64_t expected_encoded_val; uint64_t expected_encoded_val;
if (col_size == 8) { if (col_size == 8) {
expected_encoded_val = 0x0807060504030201; expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
} else if (col_size == 4) { } else if (col_size == 4) {
expected_encoded_val = 0x08070605; expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
} }
uint64_t encoded_val = 0; uint64_t encoded_val = 0;
for (int i = 0; i < row_count; ++i) { for (int i = 0; i < row_count; ++i) {
@ -122,9 +122,9 @@ TEST_P(ColumnAwareEncodingTestWithSize, RleEncodeDecode) {
memcpy(&encoded_val, encoded_data_ptr, col_size); memcpy(&encoded_val, encoded_data_ptr, col_size);
uint64_t expected_encoded_val; uint64_t expected_encoded_val;
if (col_size == 8) { if (col_size == 8) {
expected_encoded_val = 0x0807060504030201; expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708;
} else if (col_size == 4) { } else if (col_size == 4) {
expected_encoded_val = 0x08070605; expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000;
} }
// Check correctness of encoded value // Check correctness of encoded value
ASSERT_EQ(expected_encoded_val, encoded_val); ASSERT_EQ(expected_encoded_val, encoded_val);
@ -158,8 +158,8 @@ TEST_P(ColumnAwareEncodingTestWithSize, DeltaEncodeDecode) {
std::unique_ptr<ColBufEncoder> col_buf_encoder( std::unique_ptr<ColBufEncoder> col_buf_encoder(
new FixedLengthColBufEncoder(col_size, kColDeltaVarint, false, true)); new FixedLengthColBufEncoder(col_size, kColDeltaVarint, false, true));
std::string str_buf; std::string str_buf;
uint64_t base_val1 = 0x0102030405060708; uint64_t base_val1 = port::kLittleEndian ? 0x0102030405060708 : 0x0807060504030201;
uint64_t base_val2 = 0x0202030405060708; uint64_t base_val2 = port::kLittleEndian ? 0x0202030405060708 : 0x0807060504030202;
uint64_t val1 = 0, val2 = 0; uint64_t val1 = 0, val2 = 0;
memcpy(&val1, &base_val1, col_size); memcpy(&val1, &base_val1, col_size);
memcpy(&val2, &base_val2, col_size); memcpy(&val2, &base_val2, col_size);
@ -180,7 +180,7 @@ TEST_P(ColumnAwareEncodingTestWithSize, DeltaEncodeDecode) {
if (col_size == 8) { if (col_size == 8) {
varint_len = 9; varint_len = 9;
} else if (col_size == 4) { } else if (col_size == 4) {
varint_len = 5; varint_len = port::kLittleEndian ? 5 : 9;
} }
// Check encoded string length: first value is original one (val - 0), the // Check encoded string length: first value is original one (val - 0), the
// coming three are encoded as 1, -1, 1, so they should take 1 byte in varint. // coming three are encoded as 1, -1, 1, so they should take 1 byte in varint.

@ -89,7 +89,7 @@ TEST_F(HashTableTest, TestInsert) {
// verify // verify
for (uint64_t k = 0; k < max_keys; ++k) { for (uint64_t k = 0; k < max_keys; ++k) {
Node val; Node val;
port::RWMutex* rlock; port::RWMutex* rlock = nullptr;
assert(map_.Find(Node(k), &val, &rlock)); assert(map_.Find(Node(k), &val, &rlock));
rlock->ReadUnlock(); rlock->ReadUnlock();
assert(val.val_ == std::string(1000, k % 255)); assert(val.val_ == std::string(1000, k % 255));

@ -4,6 +4,11 @@
// of patent rights can be found in the PATENTS file in the same directory. // of patent rights can be found in the PATENTS file in the same directory.
// //
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#ifndef GFLAGS
#include <cstdio>
int main() { fprintf(stderr, "Please install gflags to run tools\n"); }
#else
#include <gflags/gflags.h> #include <gflags/gflags.h>
#include <atomic> #include <atomic>
#include <functional> #include <functional>
@ -349,6 +354,7 @@ int main(int argc, char** argv) {
return 0; return 0;
} }
#endif // #ifndef GFLAGS
#else #else
int main(int, char**) { return 0; } int main(int, char**) { return 0; }
#endif #endif

Loading…
Cancel
Save