From 04d58970cb5e5fbffafa2ccfee42fc479cdd48f0 Mon Sep 17 00:00:00 2001 From: Tomas Kolda Date: Fri, 21 Apr 2017 20:41:37 -0700 Subject: [PATCH] AIX and Solaris Sparc Support Summary: Replacement of #2147 The change was squashed due to a lot of conflicts. Closes https://github.com/facebook/rocksdb/pull/2194 Differential Revision: D4929799 Pulled By: siying fbshipit-source-id: 5cd49c254737a1d5ac13f3c035f128e86524c581 --- CMakeLists.txt | 15 ++- INSTALL.md | 27 ++++++ Makefile | 78 ++++++++++----- build_tools/build_detect_platform | 25 ++++- db/cuckoo_table_db_test.cc | 10 +- db/db_sst_test.cc | 2 +- db/db_test2.cc | 4 + db/external_sst_file_ingestion_job.cc | 2 + db/internal_stats.h | 1 + db/merge_test.cc | 8 +- db/perf_context_test.cc | 15 +++ db/prefix_test.cc | 68 ++++++++----- env/env_chroot.cc | 14 +++ env/env_posix.cc | 28 +++++- env/env_test.cc | 10 +- env/io_posix.cc | 10 +- env/io_posix.h | 8 +- java/CMakeLists.txt | 64 +++++++++--- java/Makefile | 13 ++- java/rocksjni/write_batch.cc | 1 - .../java/org/rocksdb/util/Environment.java | 18 +++- .../test/java/org/rocksdb/RocksDBTest.java | 2 +- .../org/rocksdb/util/EnvironmentTest.java | 19 ++-- memtable/memtablerep_bench.cc | 2 + options/options_helper.cc | 16 ++- options/options_parser.cc | 16 ++- port/port_posix.h | 8 +- port/stack_trace.cc | 2 +- table/cuckoo_table_builder_test.cc | 4 +- table/cuckoo_table_reader_test.cc | 12 ++- table/partitioned_filter_block_test.cc | 2 +- table/plain_table_index.cc | 8 +- tools/auto_sanity_test.sh | 2 +- tools/rocksdb_dump_test.sh | 2 +- tools/run_flash_bench.sh | 2 +- tools/run_leveldb.sh | 2 +- util/bloom_test.cc | 4 +- util/coding.h | 97 ++++++++++++------- util/delete_scheduler_test.cc | 4 +- util/testharness.h | 4 + util/testutil.cc | 8 +- utilities/column_aware_encoding_test.cc | 14 +-- utilities/persistent_cache/hash_table_test.cc | 2 +- .../persistent_cache_bench.cc | 6 ++ 44 files changed, 472 insertions(+), 187 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 90a30795d..c015454a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,15 @@ if(WIN32) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") endif() endif() + + option(WITH_MD_LIBRARY "build with MD" ON) + if(MSVC) + if(WITH_MD_LIBRARY) + set(RUNTIME_LIBRARY "MD") + else() + set(RUNTIME_LIBRARY "MT") + endif() + endif() else() option(WITH_SSE42 "build with SSE4.2" ON) if(WITH_SSE42) @@ -200,11 +209,11 @@ endif() if(MSVC) if((${OPTIMIZE_DEBUG} EQUAL 1)) message(STATUS "Debug optimization is enabled") - set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd") + set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /${RUNTIME_LIBRARY}d") else() - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /${RUNTIME_LIBRARY}d") endif() - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG") diff --git a/INSTALL.md b/INSTALL.md index 52055916c..cf5622875 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -106,3 +106,30 @@ makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SS * **Windows**: * For building with MS Visual Studio 13 you will need Update 4 installed. * Read and follow the instructions at CMakeLists.txt + +* **AIX 6.1** + * Install AIX Toolbox rpms with gcc + * Use these environment variables: + + export PORTABLE=1 + export CC=gcc + export AR="ar -X64" + export EXTRA_ARFLAGS=-X64 + export EXTRA_CFLAGS=-maix64 + export EXTRA_CXXFLAGS=-maix64 + export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc" + export LIBPATH=/opt/freeware/lib + export JAVA_HOME=/usr/java8_64 + export PATH=/opt/freeware/bin:$PATH + +* **Solaris Sparc** + * Install GCC 4.8.2 and higher. + * Use these environment variables: + + export CC=gcc + export EXTRA_CFLAGS=-m64 + export EXTRA_CXXFLAGS=-m64 + export EXTRA_LDFLAGS=-m64 + export PORTABLE=1 + export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc" + diff --git a/Makefile b/Makefile index db800c511..74bef9900 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,8 @@ CFLAGS += ${EXTRA_CFLAGS} CXXFLAGS += ${EXTRA_CXXFLAGS} LDFLAGS += $(EXTRA_LDFLAGS) MACHINE ?= $(shell uname -m) -ARFLAGS = rs +ARFLAGS = ${EXTRA_ARFLAGS} rs +STRIPFLAGS = -S -x # Transform parallel LOG output into something more readable. perl_command = perl -n \ @@ -154,7 +155,9 @@ missing_make_config_paths := $(shell \ $(foreach path, $(missing_make_config_paths), \ $(warning Warning: $(path) dont exist)) -ifneq ($(PLATFORM), IOS) +ifeq ($(PLATFORM), OS_AIX) +# no debug info +else ifneq ($(PLATFORM), IOS) CFLAGS += -g CXXFLAGS += -g else @@ -162,6 +165,11 @@ else OPT += -DNDEBUG endif +ifeq ($(PLATFORM), OS_AIX) +ARFLAGS = -X64 rs +STRIPFLAGS = -X64 -x +endif + ifeq ($(PLATFORM), OS_SOLARIS) PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99 endif @@ -194,6 +202,11 @@ ifdef COMPILE_WITH_TSAN LUA_PATH = endif +# AIX doesn't work with -pg +ifeq ($(PLATFORM), OS_AIX) + PROFILING_FLAGS = +endif + # USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc. ifdef COMPILE_WITH_UBSAN DISABLE_JEMALLOC=1 @@ -215,8 +228,14 @@ endif export GTEST_THROW_ON_FAILURE=1 export GTEST_HAS_EXCEPTIONS=1 GTEST_DIR = ./third-party/gtest-1.7.0/fused-src -PLATFORM_CCFLAGS += -isystem $(GTEST_DIR) -PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR) +# AIX: pre-defined system headers are surrounded by an extern "C" block +ifeq ($(PLATFORM), OS_AIX) + PLATFORM_CCFLAGS += -I$(GTEST_DIR) + PLATFORM_CXXFLAGS += -I$(GTEST_DIR) +else + PLATFORM_CCFLAGS += -isystem $(GTEST_DIR) + PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR) +endif # This (the first rule) must depend on "all". default: all @@ -571,7 +590,8 @@ ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),) # and create a randomly-named rocksdb.XXXX directory therein. # We'll use that directory in the "make check" rules. ifeq ($(TMPD),) -TMPD := $(shell f=/dev/shm; test -k $$f || f=/tmp; \ +TMPDIR := $(shell echo $${TMPDIR:-/tmp}) +TMPD := $(shell f=/dev/shm; test -k $$f || f=$(TMPDIR); \ perl -le 'use File::Temp "tempdir";' \ -e 'print tempdir("'$$f'/rocksdb.XXXX", CLEANUP => 0)') endif @@ -726,10 +746,12 @@ check: all echo "===== Running $$t"; ./$$t || exit 1; done; \ fi rm -rf $(TMPD) +ifneq ($(PLATFORM), OS_AIX) ifeq ($(filter -DROCKSDB_LITE,$(OPT)),) python tools/ldb_test.py sh tools/rocksdb_dump_test.sh endif +endif # TODO add ldb_tests check_some: $(SUBSET) @@ -1402,14 +1424,18 @@ ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PA ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar SHA256_CMD = sha256sum -ZLIB_VER = 1.2.11 -ZLIB_SHA256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 -BZIP2_VER = 1.0.6 -BZIP2_SHA256 = a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd -SNAPPY_VER = 1.1.4 -SNAPPY_SHA256 = 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057 -LZ4_VER = 1.7.5 -LZ4_SHA256 = 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e +ZLIB_VER ?= 1.2.11 +ZLIB_SHA256 ?= c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 +ZLIB_DOWNLOAD_BASE ?= http://zlib.net +BZIP2_VER ?= 1.0.6 +BZIP2_SHA256 ?= a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd +BZIP2_DOWNLOAD_BASE ?= http://www.bzip.org +SNAPPY_VER ?= 1.1.4 +SNAPPY_SHA256 ?= 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057 +SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/releases/download +LZ4_VER ?= 1.7.5 +LZ4_SHA256 ?= 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e +LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive ifeq ($(PLATFORM), OS_MACOSX) ROCKSDBJNILIB = librocksdbjni-osx.jnilib @@ -1432,47 +1458,53 @@ ifeq ($(PLATFORM), OS_SOLARIS) JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris SHA256_CMD = digest -a sha256 endif +ifeq ($(PLATFORM), OS_AIX) + JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/aix + ROCKSDBJNILIB = librocksdbjni-aix.so + EXTRACT_SOURCES = gunzip < TAR_GZ | tar xvf - + SNAPPY_MAKE_TARGET = libsnappy.la +endif libz.a: -rm -rf zlib-$(ZLIB_VER) - curl -O -L http://zlib.net/zlib-$(ZLIB_VER).tar.gz + curl -O -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \ echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \ exit 1; \ fi tar xvzf zlib-$(ZLIB_VER).tar.gz - cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC' ./configure --static && make + cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && make cp zlib-$(ZLIB_VER)/libz.a . libbz2.a: -rm -rf bzip2-$(BZIP2_VER) - curl -O -L http://www.bzip.org/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz + curl -O -L ${BZIP2_DOWNLOAD_BASE}/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \ echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \ exit 1; \ fi tar xvzf bzip2-$(BZIP2_VER).tar.gz - cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64' + cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}' cp bzip2-$(BZIP2_VER)/libbz2.a . libsnappy.a: -rm -rf snappy-$(SNAPPY_VER) - curl -O -L https://github.com/google/snappy/releases/download/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz + curl -O -L ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \ echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \ exit 1; \ fi tar xvzf snappy-$(SNAPPY_VER).tar.gz - cd snappy-$(SNAPPY_VER) && ./configure --with-pic --enable-static - cd snappy-$(SNAPPY_VER) && make + cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared + cd snappy-$(SNAPPY_VER) && make ${SNAPPY_MAKE_TARGET} cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a . liblz4.a: -rm -rf lz4-$(LZ4_VER) - curl -O -L https://github.com/lz4/lz4/archive/v$(LZ4_VER).tar.gz + curl -O -L ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \ if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \ @@ -1480,7 +1512,7 @@ liblz4.a: exit 1; \ fi tar xvzf lz4-$(LZ4_VER).tar.gz - cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2' all + cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all cp lz4-$(LZ4_VER)/lib/liblz4.a . # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries @@ -1504,7 +1536,7 @@ rocksdbjavastatic: $(java_static_libobjects) -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \ $(java_static_libobjects) $(COVERAGEFLAGS) \ $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS) - cd java/target;strip -S -x $(ROCKSDBJNILIB) + cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB) cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB) cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 55c107dc8..4358d9df9 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -128,10 +128,17 @@ case "$TARGET_OS" in ;; SunOS) PLATFORM=OS_SOLARIS - COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS" - PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64" # PORT_FILES=port/sunos/sunos_specific.cc ;; + AIX) + PLATFORM=OS_AIX + CC=gcc + COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc" + # PORT_FILES=port/aix/aix_specific.cc + ;; FreeBSD) PLATFORM=OS_FREEBSD COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD" @@ -382,6 +389,18 @@ EOF if [ "$?" = 0 ]; then PROFILING_FLAGS=-pg fi + + # Test whether sync_file_range is supported for compatibility with an old glibc + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() { + int fd = open("/dev/null", 0); + sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE); + } +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT" + fi fi # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning. @@ -427,7 +446,7 @@ elif test -z "$PORTABLE"; then COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER " elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then COMMON_FLAGS="$COMMON_FLAGS -march=z10 " - else + elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then COMMON_FLAGS="$COMMON_FLAGS -march=native " fi fi diff --git a/db/cuckoo_table_db_test.cc b/db/cuckoo_table_db_test.cc index e8f206d23..a3fd17a04 100644 --- a/db/cuckoo_table_db_test.cc +++ b/db/cuckoo_table_db_test.cc @@ -320,8 +320,14 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) { } // namespace rocksdb int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + if (rocksdb::port::kLittleEndian) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); + } + else { + fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n"); + return 0; + } } #else diff --git a/db/db_sst_test.cc b/db/db_sst_test.cc index 268c7db99..79d9f823b 100644 --- a/db/db_sst_test.cc +++ b/db/db_sst_test.cc @@ -297,7 +297,7 @@ TEST_F(DBSSTTest, RateLimitedDelete) { std::vector penalties; rocksdb::SyncPoint::GetInstance()->SetCallBack( "DeleteScheduler::BackgroundEmptyTrash:Wait", - [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); + [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); rocksdb::SyncPoint::GetInstance()->SetCallBack( "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) { // Turn timed wait into a simulated sleep diff --git a/db/db_test2.cc b/db/db_test2.cc index 51543fd8a..f109980bc 100644 --- a/db/db_test2.cc +++ b/db/db_test2.cc @@ -1516,6 +1516,7 @@ class MockPersistentCache : public PersistentCache { const size_t max_size_ = 10 * 1024; // 10KiB }; +#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented TEST_F(DBTest2, PersistentCache) { int num_iter = 80; @@ -1579,6 +1580,7 @@ TEST_F(DBTest2, PersistentCache) { } } } +#endif // !OS_SOLARIS namespace { void CountSyncPoint() { @@ -1702,6 +1704,7 @@ TEST_F(DBTest2, ReadAmpBitmap) { options.statistics->getTickerCount(READ_AMP_TOTAL_READ_BYTES)); } +#ifndef OS_SOLARIS // GetUniqueIdFromFile is not implemented TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) { if (dbname_.find("dev/shm") != std::string::npos) { // /dev/shm dont support getting a unique file id, this mean that @@ -1786,6 +1789,7 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) { ASSERT_EQ(total_useful_bytes_iter1 + total_useful_bytes_iter2, total_loaded_bytes_iter1 + total_loaded_bytes_iter2); } +#endif // !OS_SOLARIS #ifndef ROCKSDB_LITE TEST_F(DBTest2, AutomaticCompactionOverlapManualCompaction) { diff --git a/db/external_sst_file_ingestion_job.cc b/db/external_sst_file_ingestion_job.cc index ce7f9a1a0..28ccc7dc7 100644 --- a/db/external_sst_file_ingestion_job.cc +++ b/db/external_sst_file_ingestion_job.cc @@ -7,7 +7,9 @@ #include "db/external_sst_file_ingestion_job.h" +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #include #include diff --git a/db/internal_stats.h b/db/internal_stats.h index 86f376092..ee92daf52 100644 --- a/db/internal_stats.h +++ b/db/internal_stats.h @@ -49,6 +49,7 @@ struct DBPropertyInfo { extern const DBPropertyInfo* GetPropertyInfo(const Slice& property); #ifndef ROCKSDB_LITE +#undef SCORE enum class LevelStatType { INVALID = 0, NUM_FILES, diff --git a/db/merge_test.cc b/db/merge_test.cc index 5d8be99f2..dddbb4e22 100644 --- a/db/merge_test.cc +++ b/db/merge_test.cc @@ -141,7 +141,9 @@ class Counters { // mapped to a levedb Put bool set(const std::string& key, uint64_t value) { // just treat the internal rep of int64 as the string - Slice slice((char *)&value, sizeof(value)); + char buf[sizeof(value)]; + EncodeFixed64(buf, value); + Slice slice(buf, sizeof(value)); auto s = db_->Put(put_option_, key, slice); if (s.ok()) { @@ -375,7 +377,9 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges, Slice key("BatchSuccessiveMerge"); uint64_t merge_value = 1; - Slice merge_value_slice((char *)&merge_value, sizeof(merge_value)); + char buf[sizeof(merge_value)]; + EncodeFixed64(buf, merge_value); + Slice merge_value_slice(buf, sizeof(merge_value)); // Create the batch WriteBatch batch; diff --git a/db/perf_context_test.cc b/db/perf_context_test.cc index eac6053ed..f468a6831 100644 --- a/db/perf_context_test.cc +++ b/db/perf_context_test.cc @@ -633,18 +633,33 @@ TEST_F(PerfContextTest, MergeOperatorTime) { SetPerfLevel(kEnableTime); perf_context.Reset(); ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); +#ifdef OS_SOLARIS + for (int i = 0; i < 100; i++) { + ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); + } +#endif EXPECT_GT(perf_context.merge_operator_time_nanos, 0); ASSERT_OK(db->Flush(FlushOptions())); perf_context.Reset(); ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); +#ifdef OS_SOLARIS + for (int i = 0; i < 100; i++) { + ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); + } +#endif EXPECT_GT(perf_context.merge_operator_time_nanos, 0); ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr)); perf_context.Reset(); ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); +#ifdef OS_SOLARIS + for (int i = 0; i < 100; i++) { + ASSERT_OK(db->Get(ReadOptions(), "k1", &val)); + } +#endif EXPECT_GT(perf_context.merge_operator_time_nanos, 0); delete db; diff --git a/db/prefix_test.cc b/db/prefix_test.cc index bcd0b63e0..104acd8f4 100644 --- a/db/prefix_test.cc +++ b/db/prefix_test.cc @@ -32,6 +32,7 @@ int main() { #include "util/string_util.h" #include "util/testharness.h" #include "utilities/merge_operators.h" +#include "util/coding.h" using GFLAGS::ParseCommandLineFlags; @@ -65,12 +66,16 @@ struct TestKey { }; // return a slice backed by test_key -inline Slice TestKeyToSlice(const TestKey& test_key) { - return Slice((const char*)&test_key, sizeof(test_key)); +inline Slice TestKeyToSlice(std::string &s, const TestKey& test_key) { + s.clear(); + PutFixed64(&s, test_key.prefix); + PutFixed64(&s, test_key.sorted); + return Slice(s.c_str(), s.size()); } -inline const TestKey* SliceToTestKey(const Slice& slice) { - return (const TestKey*)slice.data(); +inline const TestKey SliceToTestKey(const Slice& slice) { + return TestKey(DecodeFixed64(slice.data()), + DecodeFixed64(slice.data() + 8)); } class TestKeyComparator : public Comparator { @@ -79,8 +84,10 @@ class TestKeyComparator : public Comparator { // Compare needs to be aware of the possibility of a and/or b is // prefix only virtual int Compare(const Slice& a, const Slice& b) const override { - const TestKey* key_a = SliceToTestKey(a); - const TestKey* key_b = SliceToTestKey(b); + const TestKey kkey_a = SliceToTestKey(a); + const TestKey kkey_b = SliceToTestKey(b); + const TestKey *key_a = &kkey_a; + const TestKey *key_b = &kkey_b; if (key_a->prefix != key_b->prefix) { if (key_a->prefix < key_b->prefix) return -1; if (key_a->prefix > key_b->prefix) return 1; @@ -111,7 +118,8 @@ class TestKeyComparator : public Comparator { } bool operator()(const TestKey& a, const TestKey& b) const { - return Compare(TestKeyToSlice(a), TestKeyToSlice(b)) < 0; + std::string sa, sb; + return Compare(TestKeyToSlice(sa, a), TestKeyToSlice(sb, b)) < 0; } virtual const char* Name() const override { @@ -128,30 +136,35 @@ namespace { void PutKey(DB* db, WriteOptions write_options, uint64_t prefix, uint64_t suffix, const Slice& value) { TestKey test_key(prefix, suffix); - Slice key = TestKeyToSlice(test_key); + std::string s; + Slice key = TestKeyToSlice(s, test_key); ASSERT_OK(db->Put(write_options, key, value)); } void PutKey(DB* db, WriteOptions write_options, const TestKey& test_key, const Slice& value) { - Slice key = TestKeyToSlice(test_key); + std::string s; + Slice key = TestKeyToSlice(s, test_key); ASSERT_OK(db->Put(write_options, key, value)); } void MergeKey(DB* db, WriteOptions write_options, const TestKey& test_key, const Slice& value) { - Slice key = TestKeyToSlice(test_key); + std::string s; + Slice key = TestKeyToSlice(s, test_key); ASSERT_OK(db->Merge(write_options, key, value)); } void DeleteKey(DB* db, WriteOptions write_options, const TestKey& test_key) { - Slice key = TestKeyToSlice(test_key); + std::string s; + Slice key = TestKeyToSlice(s, test_key); ASSERT_OK(db->Delete(write_options, key)); } void SeekIterator(Iterator* iter, uint64_t prefix, uint64_t suffix) { TestKey test_key(prefix, suffix); - Slice key = TestKeyToSlice(test_key); + std::string s; + Slice key = TestKeyToSlice(s, test_key); iter->Seek(key); } @@ -160,7 +173,8 @@ const std::string kNotFoundResult = "NOT_FOUND"; std::string Get(DB* db, const ReadOptions& read_options, uint64_t prefix, uint64_t suffix) { TestKey test_key(prefix, suffix); - Slice key = TestKeyToSlice(test_key); + std::string s2; + Slice key = TestKeyToSlice(s2, test_key); std::string result; Status s = db->Get(read_options, key, &result); @@ -527,7 +541,9 @@ TEST_F(PrefixTest, PrefixValid) { PutKey(db.get(), write_options, 12345, 9, v19); PutKey(db.get(), write_options, 12346, 8, v16); db->Flush(FlushOptions()); - db->Delete(write_options, TestKeyToSlice(TestKey(12346, 8))); + TestKey test_key(12346, 8); + std::string s; + db->Delete(write_options, TestKeyToSlice(s, test_key)); db->Flush(FlushOptions()); read_options.prefix_same_as_start = true; std::unique_ptr iter(db->NewIterator(read_options)); @@ -583,7 +599,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) { for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) { TestKey test_key(prefix, sorted); - Slice key = TestKeyToSlice(test_key); + std::string s; + Slice key = TestKeyToSlice(s, test_key); std::string value(FLAGS_value_size, 0); perf_context.Reset(); @@ -605,7 +622,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) { for (auto prefix : prefixes) { TestKey test_key(prefix, FLAGS_items_per_prefix / 2); - Slice key = TestKeyToSlice(test_key); + std::string s; + Slice key = TestKeyToSlice(s, test_key); std::string value = "v" + ToString(0); perf_context.Reset(); @@ -639,7 +657,8 @@ TEST_F(PrefixTest, DynamicPrefixIterator) { prefix < FLAGS_total_prefixes + 10000; prefix++) { TestKey test_key(prefix, 0); - Slice key = TestKeyToSlice(test_key); + std::string s; + Slice key = TestKeyToSlice(s, test_key); perf_context.Reset(); StopWatchNano timer(Env::Default(), true); @@ -725,8 +744,8 @@ TEST_F(PrefixTest, PrefixSeekModePrev) { ASSERT_TRUE(iter->Valid()); if (FLAGS_enable_print) { std::cout << "round " << prefix - << " iter: " << SliceToTestKey(iter->key())->prefix - << SliceToTestKey(iter->key())->sorted + << " iter: " << SliceToTestKey(iter->key()).prefix + << SliceToTestKey(iter->key()).sorted << " | map: " << it->first.prefix << it->first.sorted << " | " << iter->value().ToString() << " " << it->second << std::endl; } @@ -747,16 +766,16 @@ TEST_F(PrefixTest, PrefixSeekModePrev) { } } if (!iter->Valid() || - SliceToTestKey(iter->key())->prefix != stored_prefix) { + SliceToTestKey(iter->key()).prefix != stored_prefix) { break; } - stored_prefix = SliceToTestKey(iter->key())->prefix; + stored_prefix = SliceToTestKey(iter->key()).prefix; ASSERT_TRUE(iter->Valid()); ASSERT_NE(it, whole_map.end()); ASSERT_EQ(iter->value(), it->second); if (FLAGS_enable_print) { - std::cout << "iter: " << SliceToTestKey(iter->key())->prefix - << SliceToTestKey(iter->key())->sorted + std::cout << "iter: " << SliceToTestKey(iter->key()).prefix + << SliceToTestKey(iter->key()).sorted << " | map: " << it->first.prefix << it->first.sorted << " | " << iter->value().ToString() << " " << it->second << std::endl; @@ -810,7 +829,8 @@ TEST_F(PrefixTest, PrefixSeekModePrev3) { options.write_buffer_size = 1024 * 1024; std::string v14("v14"); TestKey upper_bound_key = TestKey(1, 5); - Slice upper_bound = TestKeyToSlice(upper_bound_key); + std::string s; + Slice upper_bound = TestKeyToSlice(s, upper_bound_key); { DestroyDB(kDbName, Options()); diff --git a/env/env_chroot.cc b/env/env_chroot.cc index 9b7c205d1..90583a234 100644 --- a/env/env_chroot.cc +++ b/env/env_chroot.cc @@ -24,11 +24,18 @@ class ChrootEnv : public EnvWrapper { public: ChrootEnv(Env* base_env, const std::string& chroot_dir) : EnvWrapper(base_env) { +#if defined(OS_AIX) + char resolvedName[PATH_MAX]; + char* real_chroot_dir = realpath(chroot_dir.c_str(), resolvedName); +#else char* real_chroot_dir = realpath(chroot_dir.c_str(), nullptr); +#endif // chroot_dir must exist so realpath() returns non-nullptr. assert(real_chroot_dir != nullptr); chroot_dir_ = real_chroot_dir; +#if !defined(OS_AIX) free(real_chroot_dir); +#endif } virtual Status NewSequentialFile(const std::string& fname, @@ -258,7 +265,12 @@ class ChrootEnv : public EnvWrapper { } std::pair res; res.second = chroot_dir_ + path; +#if defined(OS_AIX) + char resolvedName[PATH_MAX]; + char* normalized_path = realpath(res.second.c_str(), resolvedName); +#else char* normalized_path = realpath(res.second.c_str(), nullptr); +#endif if (normalized_path == nullptr) { res.first = Status::NotFound(res.second, strerror(errno)); } else if (strlen(normalized_path) < chroot_dir_.size() || @@ -269,7 +281,9 @@ class ChrootEnv : public EnvWrapper { } else { res.first = Status::OK(); } +#if !defined(OS_AIX) free(normalized_path); +#endif return res; } diff --git a/env/env_posix.cc b/env/env_posix.cc index 02f82e92e..2b29b2b85 100644 --- a/env/env_posix.cc +++ b/env/env_posix.cc @@ -159,7 +159,7 @@ class PosixEnv : public Env { #ifdef ROCKSDB_LITE return Status::IOError(fname, "Direct I/O not supported in RocksDB lite"); #endif // !ROCKSDB_LITE -#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) +#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS) flags |= O_DIRECT; #endif } @@ -206,7 +206,7 @@ class PosixEnv : public Env { #ifdef ROCKSDB_LITE return Status::IOError(fname, "Direct I/O not supported in RocksDB lite"); #endif // !ROCKSDB_LITE -#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) +#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS) flags |= O_DIRECT; TEST_SYNC_POINT_CALLBACK("NewRandomAccessFile:O_DIRECT", &flags); #endif @@ -271,7 +271,7 @@ class PosixEnv : public Env { return Status::IOError(fname, "Direct I/O not supported in RocksDB lite"); #endif // ROCKSDB_LITE flags |= O_WRONLY; -#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) +#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS) flags |= O_DIRECT; #endif TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags); @@ -312,6 +312,14 @@ class PosixEnv : public Env { s = IOError(fname, errno); return s; } +#elif defined(OS_SOLARIS) + if (directio(fd, DIRECTIO_ON) == -1) { + if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON + close(fd); + s = IOError(fname, errno); + return s; + } + } #endif result->reset(new PosixWritableFile(fname, fd, options)); } else { @@ -338,7 +346,7 @@ class PosixEnv : public Env { return Status::IOError(fname, "Direct I/O not supported in RocksDB lite"); #endif // !ROCKSDB_LITE flags |= O_WRONLY; -#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) +#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS) flags |= O_DIRECT; #endif TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags); @@ -385,6 +393,14 @@ class PosixEnv : public Env { s = IOError(fname, errno); return s; } +#elif defined(OS_SOLARIS) + if (directio(fd, DIRECTIO_ON) == -1) { + if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON + close(fd); + s = IOError(fname, errno); + return s; + } + } #endif result->reset(new PosixWritableFile(fname, fd, options)); } else { @@ -668,10 +684,12 @@ class PosixEnv : public Env { } virtual uint64_t NowNanos() override { -#if defined(OS_LINUX) || defined(OS_FREEBSD) +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_AIX) struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return static_cast(ts.tv_sec) * 1000000000 + ts.tv_nsec; +#elif defined(OS_SOLARIS) + return gethrtime(); #elif defined(__MACH__) clock_serv_t cclock; mach_timespec_t ts; diff --git a/env/env_test.cc b/env/env_test.cc index 08deba72f..6409d1f03 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -958,7 +958,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) { // Create file. { unique_ptr wfile; -#if !defined(OS_MACOSX) && !defined(OS_WIN) +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX) if (soptions.use_direct_writes) { soptions.use_direct_writes = false; } @@ -974,7 +974,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) { unique_ptr file; auto scratch = NewAligned(kSectorSize, 0); Slice result; -#if !defined(OS_MACOSX) && !defined(OS_WIN) +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX) if (soptions.use_direct_reads) { soptions.use_direct_reads = false; } @@ -991,7 +991,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) { unique_ptr file; auto scratch = NewAligned(kSectorSize, 0); Slice result; -#if !defined(OS_MACOSX) && !defined(OS_WIN) +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX) if (soptions.use_direct_reads) { soptions.use_direct_reads = false; } @@ -1136,7 +1136,7 @@ TEST_P(EnvPosixTestWithParam, Preallocation) { unique_ptr srcfile; EnvOptions soptions; soptions.use_direct_reads = soptions.use_direct_writes = direct_io_; -#if !defined(OS_MACOSX) && !defined(OS_WIN) +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX) if (soptions.use_direct_writes) { rocksdb::SyncPoint::GetInstance()->SetCallBack( "NewWritableFile:O_DIRECT", [&](void* arg) { @@ -1198,7 +1198,7 @@ TEST_P(EnvPosixTestWithParam, ConsistentChildrenAttributes) { oss << test::TmpDir(env_) << "/testfile_" << i; const std::string path = oss.str(); unique_ptr file; -#if !defined(OS_MACOSX) && !defined(OS_WIN) +#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && !defined(OS_AIX) if (soptions.use_direct_writes) { rocksdb::SyncPoint::GetInstance()->SetCallBack( "NewWritableFile:O_DIRECT", [&](void* arg) { diff --git a/env/io_posix.cc b/env/io_posix.cc index fee969022..f994e8c34 100644 --- a/env/io_posix.cc +++ b/env/io_posix.cc @@ -267,7 +267,7 @@ size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) { } #endif -#if defined(OS_MACOSX) +#if defined(OS_MACOSX) || defined(OS_AIX) size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) { if (max_size < kMaxVarint64Length * 3) { return 0; @@ -336,7 +336,7 @@ Status PosixRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result, return s; } -#if defined(OS_LINUX) || defined(OS_MACOSX) +#if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX) size_t PosixRandomAccessFile::GetUniqueId(char* id, size_t max_size) const { return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size); } @@ -848,7 +848,7 @@ Status PosixWritableFile::Allocate(uint64_t offset, uint64_t len) { } #endif -#ifdef OS_LINUX +#ifdef ROCKSDB_RANGESYNC_PRESENT Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) { assert(offset <= std::numeric_limits::max()); assert(nbytes <= std::numeric_limits::max()); @@ -859,7 +859,9 @@ Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) { return IOError(filename_, errno); } } +#endif +#ifdef OS_LINUX size_t PosixWritableFile::GetUniqueId(char* id, size_t max_size) const { return PosixHelper::GetUniqueIdFromFile(fd_, id, max_size); } @@ -961,9 +963,11 @@ Status PosixRandomRWFile::Close() { PosixDirectory::~PosixDirectory() { close(fd_); } Status PosixDirectory::Fsync() { +#ifndef OS_AIX if (fsync(fd_) == -1) { return IOError("directory", errno); } +#endif return Status::OK(); } } // namespace rocksdb diff --git a/env/io_posix.h b/env/io_posix.h index 0feeec8b8..351d6d7cb 100644 --- a/env/io_posix.h +++ b/env/io_posix.h @@ -15,7 +15,7 @@ // For non linux platform, the following macros are used only as place // holder. -#if !(defined OS_LINUX) && !(defined CYGWIN) +#if !(defined OS_LINUX) && !(defined CYGWIN) && !(defined OS_AIX) #define POSIX_FADV_NORMAL 0 /* [MC1] no further special treatment */ #define POSIX_FADV_RANDOM 1 /* [MC1] expect random page refs */ #define POSIX_FADV_SEQUENTIAL 2 /* [MC1] expect sequential page refs */ @@ -79,7 +79,7 @@ class PosixRandomAccessFile : public RandomAccessFile { virtual Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const override; -#if defined(OS_LINUX) || defined(OS_MACOSX) +#if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX) virtual size_t GetUniqueId(char* id, size_t max_size) const override; #endif virtual void Hint(AccessPattern pattern) override; @@ -126,8 +126,10 @@ class PosixWritableFile : public WritableFile { #ifdef ROCKSDB_FALLOCATE_PRESENT virtual Status Allocate(uint64_t offset, uint64_t len) override; #endif -#ifdef OS_LINUX +#ifdef ROCKSDB_RANGESYNC_PRESENT virtual Status RangeSync(uint64_t offset, uint64_t nbytes) override; +#endif +#ifdef OS_LINUX virtual size_t GetUniqueId(char* id, size_t max_size) const override; #endif }; diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index 4e780523d..92c4a0e97 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -1,25 +1,31 @@ cmake_minimum_required(VERSION 2.6) set(JNI_NATIVE_SOURCES - rocksjni/backupenginejni.cc rocksjni/backupablejni.cc + rocksjni/backupenginejni.cc rocksjni/checkpoint.cc + rocksjni/clock_cache.cc rocksjni/columnfamilyhandle.cc rocksjni/compaction_filter.cc + rocksjni/compaction_options_fifo.cc + rocksjni/compaction_options_universal.cc rocksjni/comparator.cc rocksjni/comparatorjnicallback.cc + rocksjni/compression_options.cc rocksjni/env.cc rocksjni/env_options.cc rocksjni/external_sst_file_info.cc rocksjni/filter.cc rocksjni/iterator.cc rocksjni/loggerjnicallback.cc + rocksjni/lru_cache.cc rocksjni/memtablejni.cc rocksjni/merge_operator.cc rocksjni/options.cc rocksjni/ratelimiterjni.cc rocksjni/remove_emptyvalue_compactionfilterjni.cc rocksjni/restorejni.cc + rocksjni/rocksdb_exception_test.cc rocksjni/rocksjni.cc rocksjni/slice.cc rocksjni/snapshot.cc @@ -29,38 +35,50 @@ set(JNI_NATIVE_SOURCES rocksjni/transaction_log.cc rocksjni/ttl.cc rocksjni/write_batch.cc - rocksjni/writebatchhandlerjnicallback.cc - rocksjni/write_batch_with_index.cc rocksjni/write_batch_test.cc + rocksjni/write_batch_with_index.cc + rocksjni/writebatchhandlerjnicallback.cc ) set(NATIVE_JAVA_CLASSES org.rocksdb.AbstractCompactionFilter org.rocksdb.AbstractComparator + org.rocksdb.AbstractImmutableNativeReference + org.rocksdb.AbstractNativeReference + org.rocksdb.AbstractRocksIterator org.rocksdb.AbstractSlice - org.rocksdb.BackupEngine + org.rocksdb.AbstractWriteBatch org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions + org.rocksdb.BackupEngine + org.rocksdb.BackupEngineTest org.rocksdb.BlockBasedTableConfig org.rocksdb.BloomFilter + org.rocksdb.Cache org.rocksdb.Checkpoint + org.rocksdb.ClockCache org.rocksdb.ColumnFamilyHandle org.rocksdb.ColumnFamilyOptions + org.rocksdb.CompactionOptionsFIFO + org.rocksdb.CompactionOptionsUniversal org.rocksdb.Comparator org.rocksdb.ComparatorOptions + org.rocksdb.CompressionOptions org.rocksdb.DBOptions org.rocksdb.DirectComparator org.rocksdb.DirectSlice org.rocksdb.Env org.rocksdb.EnvOptions org.rocksdb.ExternalSstFileInfo - org.rocksdb.FlushOptions org.rocksdb.Filter - org.rocksdb.GenericRateLimiterConfig + org.rocksdb.FlushOptions org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.Logger + org.rocksdb.LRUCache + org.rocksdb.MemTableConfig org.rocksdb.MergeOperator + org.rocksdb.NativeLibraryLoader org.rocksdb.Options org.rocksdb.PlainTableConfig org.rocksdb.RateLimiter @@ -69,25 +87,31 @@ set(NATIVE_JAVA_CLASSES org.rocksdb.RestoreBackupableDB org.rocksdb.RestoreOptions org.rocksdb.RocksDB + org.rocksdb.RocksDBExceptionTest org.rocksdb.RocksEnv org.rocksdb.RocksIterator + org.rocksdb.RocksIteratorInterface org.rocksdb.RocksMemEnv + org.rocksdb.RocksMutableObject + org.rocksdb.RocksObject org.rocksdb.SkipListMemTableConfig org.rocksdb.Slice + org.rocksdb.Snapshot + org.rocksdb.SnapshotTest org.rocksdb.SstFileWriter org.rocksdb.Statistics + org.rocksdb.StringAppendOperator + org.rocksdb.TableFormatConfig org.rocksdb.TransactionLogIterator org.rocksdb.TtlDB org.rocksdb.VectorMemTableConfig - org.rocksdb.Snapshot - org.rocksdb.StringAppendOperator + org.rocksdb.WBWIRocksIterator org.rocksdb.WriteBatch org.rocksdb.WriteBatch.Handler - org.rocksdb.WriteOptions - org.rocksdb.WriteBatchWithIndex - org.rocksdb.WBWIRocksIterator org.rocksdb.WriteBatchTest org.rocksdb.WriteBatchTestInternalHelper + org.rocksdb.WriteBatchWithIndex + org.rocksdb.WriteOptions ) include_directories($ENV{JAVA_HOME}/include) @@ -111,9 +135,17 @@ if(NOT EXISTS ${JAVA_TEST_LIBDIR}) file(MAKE_DIRECTORY mkdir ${JAVA_TEST_LIBDIR}) endif() +if (DEFINED CUSTOM_REPO_URL) + set(SEARCH_REPO_URL ${CUSTOM_REPO_URL}/) + set(CENTRAL_REPO_URL ${CUSTOM_REPO_URL}/) +else () + set(SEARCH_REPO_URL "http://search.maven.org/remotecontent?filepath=") + set(CENTRAL_REPO_URL "http://central.maven.org/maven2/") +endif() + if(NOT EXISTS ${JAVA_JUNIT_JAR}) message("Downloading ${JAVA_JUNIT_JAR}") - file(DOWNLOAD http://search.maven.org/remotecontent?filepath=junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${SEARCH_REPO_URL}junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}") @@ -122,7 +154,7 @@ if(NOT EXISTS ${JAVA_JUNIT_JAR}) endif() if(NOT EXISTS ${JAVA_HAMCR_JAR}) message("Downloading ${JAVA_HAMCR_JAR}") - file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${SEARCH_REPO_URL}org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}") @@ -131,7 +163,7 @@ if(NOT EXISTS ${JAVA_HAMCR_JAR}) endif() if(NOT EXISTS ${JAVA_MOCKITO_JAR}) message("Downloading ${JAVA_MOCKITO_JAR}") - file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${SEARCH_REPO_URL}org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}") @@ -140,7 +172,7 @@ if(NOT EXISTS ${JAVA_MOCKITO_JAR}) endif() if(NOT EXISTS ${JAVA_CGLIB_JAR}) message("Downloading ${JAVA_CGLIB_JAR}") - file(DOWNLOAD http://search.maven.org/remotecontent?filepath=cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${SEARCH_REPO_URL}cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}") @@ -149,7 +181,7 @@ if(NOT EXISTS ${JAVA_CGLIB_JAR}) endif() if(NOT EXISTS ${JAVA_ASSERTJ_JAR}) message("Downloading ${JAVA_ASSERTJ_JAR}") - file(DOWNLOAD http://central.maven.org/maven2/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) + file(DOWNLOAD ${CENTRAL_REPO_URL}org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus) list(GET downloadStatus 0 error_code) if(NOT error_code EQUAL 0) message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}") diff --git a/java/Makefile b/java/Makefile index 52e802d5a..ab2054602 100644 --- a/java/Makefile +++ b/java/Makefile @@ -157,6 +157,9 @@ ifneq ($(DEBUG_LEVEL),0) JAVA_ARGS = -ea -Xcheck:jni endif +SEARCH_REPO_URL?=http://search.maven.org/remotecontent?filepath= +CENTRAL_REPO_URL?=http://central.maven.org/maven2/ + clean: $(AM_V_at)rm -rf include/* $(AM_V_at)rm -rf test-libs/ @@ -198,11 +201,11 @@ column_family_sample: java resolve_test_deps: test -d "$(JAVA_TEST_LIBDIR)" || mkdir -p "$(JAVA_TEST_LIBDIR)" - test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) http://search.maven.org/remotecontent?filepath=junit/junit/4.12/junit-4.12.jar - test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar - test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" http://search.maven.org/remotecontent?filepath=org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar - test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" http://search.maven.org/remotecontent?filepath=cglib/cglib/2.2.2/cglib-2.2.2.jar - test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" http://central.maven.org/maven2/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar + test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) $(SEARCH_REPO_URL)junit/junit/4.12/junit-4.12.jar + test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) $(SEARCH_REPO_URL)org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar + test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" $(SEARCH_REPO_URL)org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar + test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" $(SEARCH_REPO_URL)cglib/cglib/2.2.2/cglib-2.2.2.jar + test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" $(CENTRAL_REPO_URL)org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar java_test: java resolve_test_deps $(AM_V_GEN)mkdir -p $(TEST_CLASSES) diff --git a/java/rocksjni/write_batch.cc b/java/rocksjni/write_batch.cc index bef1e8d0c..502e1849d 100644 --- a/java/rocksjni/write_batch.cc +++ b/java/rocksjni/write_batch.cc @@ -21,7 +21,6 @@ #include "rocksjni/writebatchhandlerjnicallback.h" #include "table/scoped_arena_iterator.h" #include "util/logging.h" -#include "util/testharness.h" /* * Class: org_rocksdb_WriteBatch diff --git a/java/src/main/java/org/rocksdb/util/Environment.java b/java/src/main/java/org/rocksdb/util/Environment.java index 061d2ff91..f84e14bc1 100644 --- a/java/src/main/java/org/rocksdb/util/Environment.java +++ b/java/src/main/java/org/rocksdb/util/Environment.java @@ -16,17 +16,23 @@ public class Environment { return (OS.contains("mac")); } + public static boolean isAix() { + return OS.contains("aix"); + } + public static boolean isUnix() { - return (OS.contains("nix") || - OS.contains("nux") || - OS.contains("aix")); + return OS.contains("nix") || + OS.contains("nux"); } public static boolean isSolaris() { - return OS.contains("sunos"); + return OS.contains("sunos"); } public static boolean is64Bit() { + if (ARCH.indexOf("sparcv9") >= 0) { + return true; + } return (ARCH.indexOf("64") > 0); } @@ -48,6 +54,8 @@ public class Environment { } } else if (isMac()) { return String.format("%sjni-osx", name); + } else if (isAix() && is64Bit()) { + return String.format("%sjni-aix64", name); } else if (isSolaris()) { final String arch = is64Bit() ? "64" : "32"; return String.format("%sjni-solaris%s", name, arch); @@ -63,7 +71,7 @@ public class Environment { } private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) { - if (isUnix() || isSolaris()) { + if (isUnix() || isAix() || isSolaris()) { return libraryFileName + ".so"; } else if (isMac()) { return libraryFileName + (shared ? ".dylib" : ".jnilib"); diff --git a/java/src/test/java/org/rocksdb/RocksDBTest.java b/java/src/test/java/org/rocksdb/RocksDBTest.java index b5c1edd25..e95497478 100644 --- a/java/src/test/java/org/rocksdb/RocksDBTest.java +++ b/java/src/test/java/org/rocksdb/RocksDBTest.java @@ -53,7 +53,7 @@ public class RocksDBTest { } catch (final RocksDBException e) { assertThat(e.getStatus().getCode()).isEqualTo(Status.Code.IOError); assertThat(e.getStatus().getSubCode()).isEqualTo(Status.SubCode.None); - assertThat(e.getStatus().getState()).startsWith("lock "); + assertThat(e.getStatus().getState()).contains("lock "); } } } diff --git a/java/src/test/java/org/rocksdb/util/EnvironmentTest.java b/java/src/test/java/org/rocksdb/util/EnvironmentTest.java index 2de1c45f7..85e0c632c 100644 --- a/java/src/test/java/org/rocksdb/util/EnvironmentTest.java +++ b/java/src/test/java/org/rocksdb/util/EnvironmentTest.java @@ -1,4 +1,4 @@ -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// Copyright (c) 2014, Facebook, Inc. All rights reserved. // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. @@ -70,15 +70,16 @@ public class EnvironmentTest { isEqualTo("librocksdbjni-linux32.so"); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.so"); + } + + @Test(expected = UnsupportedOperationException.class) + public void aix32() { // AIX setEnvironmentClassFields("aix", "32"); assertThat(Environment.isWindows()).isFalse(); assertThat(Environment.getJniLibraryExtension()). isEqualTo(".so"); - assertThat(Environment.getJniLibraryFileName("rocksdb")). - isEqualTo("librocksdbjni-linux32.so"); - assertThat(Environment.getSharedLibraryFileName("rocksdb")). - isEqualTo("librocksdbjni.so"); + Environment.getJniLibraryFileName("rocksdb"); } @Test @@ -106,7 +107,7 @@ public class EnvironmentTest { assertThat(Environment.getJniLibraryExtension()). isEqualTo(".so"); assertThat(Environment.getJniLibraryFileName("rocksdb")). - isEqualTo("librocksdbjni-linux64.so"); + isEqualTo("librocksdbjni-aix64.so"); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.so"); } @@ -129,12 +130,6 @@ public class EnvironmentTest { isEqualTo("librocksdbjni.dll"); } - @Test(expected = UnsupportedOperationException.class) - public void win32(){ - setEnvironmentClassFields("win", "32"); - Environment.getJniLibraryFileName("rocksdb"); - } - private void setEnvironmentClassFields(String osName, String osArch) { setEnvironmentClassField(OS_FIELD_NAME, osName); diff --git a/memtable/memtablerep_bench.cc b/memtable/memtablerep_bench.cc index 7eaffab91..9fdb63aa7 100644 --- a/memtable/memtablerep_bench.cc +++ b/memtable/memtablerep_bench.cc @@ -7,7 +7,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #ifndef GFLAGS #include diff --git a/options/options_helper.cc b/options/options_helper.cc index d66844c0e..206eb2e10 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -320,10 +320,10 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, *reinterpret_cast(opt_address) = ParseUint32(value); break; case OptionType::kUInt64T: - *reinterpret_cast(opt_address) = ParseUint64(value); + PutUnaligned(reinterpret_cast(opt_address), ParseUint64(value)); break; case OptionType::kSizeT: - *reinterpret_cast(opt_address) = ParseSizeT(value); + PutUnaligned(reinterpret_cast(opt_address), ParseSizeT(value)); break; case OptionType::kString: *reinterpret_cast(opt_address) = value; @@ -404,10 +404,18 @@ bool SerializeSingleOptionHelper(const char* opt_address, *value = ToString(*(reinterpret_cast(opt_address))); break; case OptionType::kUInt64T: - *value = ToString(*(reinterpret_cast(opt_address))); + { + uint64_t v; + GetUnaligned(reinterpret_cast(opt_address), &v); + *value = ToString(v); + } break; case OptionType::kSizeT: - *value = ToString(*(reinterpret_cast(opt_address))); + { + size_t v; + GetUnaligned(reinterpret_cast(opt_address), &v); + *value = ToString(v); + } break; case OptionType::kDouble: *value = ToString(*(reinterpret_cast(opt_address))); diff --git a/options/options_parser.cc b/options/options_parser.cc index b9db968a0..09f3e1cbf 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -528,11 +528,19 @@ bool AreEqualOptions( return (*reinterpret_cast(offset1) == *reinterpret_cast(offset2)); case OptionType::kUInt64T: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); + { + uint64_t v1, v2; + GetUnaligned(reinterpret_cast(offset1), &v1); + GetUnaligned(reinterpret_cast(offset2), &v2); + return (v1 == v2); + } case OptionType::kSizeT: - return (*reinterpret_cast(offset1) == - *reinterpret_cast(offset2)); + { + size_t v1, v2; + GetUnaligned(reinterpret_cast(offset1), &v1); + GetUnaligned(reinterpret_cast(offset2), &v2); + return (v1 == v2); + } case OptionType::kString: return (*reinterpret_cast(offset1) == *reinterpret_cast(offset2)); diff --git a/port/port_posix.h b/port/port_posix.h index 05e34aca4..031879ea0 100644 --- a/port/port_posix.h +++ b/port/port_posix.h @@ -35,6 +35,12 @@ #else #define PLATFORM_IS_LITTLE_ENDIAN false #endif + #include +#elif defined(OS_AIX) + #include + #include + #define PLATFORM_IS_LITTLE_ENDIAN (BYTE_ORDER == LITTLE_ENDIAN) + #include #elif defined(OS_FREEBSD) || defined(OS_OPENBSD) || defined(OS_NETBSD) || \ defined(OS_DRAGONFLYBSD) || defined(OS_ANDROID) #include @@ -56,7 +62,7 @@ #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\ - defined(OS_ANDROID) || defined(CYGWIN) + defined(OS_ANDROID) || defined(CYGWIN) || defined(OS_AIX) // Use fread/fwrite/fflush on platforms without _unlocked variants #define fread_unlocked fread #define fwrite_unlocked fwrite diff --git a/port/stack_trace.cc b/port/stack_trace.cc index 74996e836..c8ad789c9 100644 --- a/port/stack_trace.cc +++ b/port/stack_trace.cc @@ -6,7 +6,7 @@ #include "port/stack_trace.h" #if defined(ROCKSDB_LITE) || !(defined(ROCKSDB_BACKTRACE) || defined(OS_MACOSX)) || \ - defined(CYGWIN) || defined(OS_FREEBSD) + defined(CYGWIN) || defined(OS_FREEBSD) || defined(OS_SOLARIS) // noop diff --git a/table/cuckoo_table_builder_test.cc b/table/cuckoo_table_builder_test.cc index bc8514ff2..b50a40b21 100644 --- a/table/cuckoo_table_builder_test.cc +++ b/table/cuckoo_table_builder_test.cc @@ -66,8 +66,8 @@ class CuckooBuilderTest : public testing::Test { ASSERT_EQ(expected_unused_bucket.substr(0, props->fixed_key_len), unused_key); - uint32_t value_len_found = - *reinterpret_cast(props->user_collected_properties[ + uint64_t value_len_found = + *reinterpret_cast(props->user_collected_properties[ CuckooTablePropertyNames::kValueLength].data()); ASSERT_EQ(values.empty() ? 0 : values[0].size(), value_len_found); ASSERT_EQ(props->raw_value_size, values.size()*value_len_found); diff --git a/table/cuckoo_table_reader_test.cc b/table/cuckoo_table_reader_test.cc index 953aadc72..e7d958d49 100644 --- a/table/cuckoo_table_reader_test.cc +++ b/table/cuckoo_table_reader_test.cc @@ -544,9 +544,15 @@ TEST_F(CuckooReaderTest, TestReadPerformance) { } // namespace rocksdb int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - ParseCommandLineFlags(&argc, &argv, true); - return RUN_ALL_TESTS(); + if (rocksdb::port::kLittleEndian) { + ::testing::InitGoogleTest(&argc, argv); + ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); + } + else { + fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n"); + return 0; + } } #endif // GFLAGS. diff --git a/table/partitioned_filter_block_test.cc b/table/partitioned_filter_block_test.cc index 3f320d48e..123cb17a5 100644 --- a/table/partitioned_filter_block_test.cc +++ b/table/partitioned_filter_block_test.cc @@ -56,7 +56,7 @@ class PartitionedFilterBlockTest : public testing::Test { int num_keys = sizeof(keys) / sizeof(*keys); uint64_t max_key_size = 0; for (int i = 1; i < num_keys; i++) { - max_key_size = std::max(max_key_size, keys[i].size()); + max_key_size = std::max(max_key_size, static_cast(keys[i].size())); } uint64_t max_index_size = num_keys * (max_key_size + 8 /*handle*/); return max_index_size; diff --git a/table/plain_table_index.cc b/table/plain_table_index.cc index 9183eeffa..07db25894 100644 --- a/table/plain_table_index.cc +++ b/table/plain_table_index.cc @@ -44,7 +44,7 @@ Status PlainTableIndex::InitFromRawData(Slice data) { PlainTableIndex::IndexSearchResult PlainTableIndex::GetOffset( uint32_t prefix_hash, uint32_t* bucket_value) const { int bucket = GetBucketIdFromHash(prefix_hash, index_size_); - *bucket_value = index_[bucket]; + GetUnaligned(index_ + bucket, bucket_value); if ((*bucket_value & kSubIndexMask) == kSubIndexMask) { *bucket_value ^= kSubIndexMask; return kSubindex; @@ -175,15 +175,15 @@ Slice PlainTableIndexBuilder::FillIndexes( switch (num_keys_for_bucket) { case 0: // No key for bucket - index[i] = PlainTableIndex::kMaxFileSize; + PutUnaligned(index + i, (uint32_t)PlainTableIndex::kMaxFileSize); break; case 1: // point directly to the file offset - index[i] = hash_to_offsets[i]->offset; + PutUnaligned(index + i, hash_to_offsets[i]->offset); break; default: // point to second level indexes. - index[i] = sub_index_offset | PlainTableIndex::kSubIndexMask; + PutUnaligned(index + i, sub_index_offset | PlainTableIndex::kSubIndexMask); char* prev_ptr = &sub_index[sub_index_offset]; char* cur_ptr = EncodeVarint32(prev_ptr, num_keys_for_bucket); sub_index_offset += static_cast(cur_ptr - prev_ptr); diff --git a/tools/auto_sanity_test.sh b/tools/auto_sanity_test.sh index 07d444efb..54577ffb7 100755 --- a/tools/auto_sanity_test.sh +++ b/tools/auto_sanity_test.sh @@ -1,4 +1,4 @@ -TMP_DIR="/tmp/rocksdb-sanity-test" +TMP_DIR="${TMPDIR:-/tmp}/rocksdb-sanity-test" if [ "$#" -lt 2 ]; then echo "usage: ./auto_sanity_test.sh [new_commit] [old_commit]" diff --git a/tools/rocksdb_dump_test.sh b/tools/rocksdb_dump_test.sh index 5c8b5c30a..2cf8c060b 100755 --- a/tools/rocksdb_dump_test.sh +++ b/tools/rocksdb_dump_test.sh @@ -1,4 +1,4 @@ -TESTDIR=`mktemp -d /tmp/rocksdb-dump-test.XXXXX` +TESTDIR=`mktemp -d ${TMPDIR:-/tmp}/rocksdb-dump-test.XXXXX` DUMPFILE="tools/sample-dump.dmp" # Verify that the sample dump file is undumpable and then redumpable. diff --git a/tools/run_flash_bench.sh b/tools/run_flash_bench.sh index fc2c9470f..76c16bb59 100755 --- a/tools/run_flash_bench.sh +++ b/tools/run_flash_bench.sh @@ -102,7 +102,7 @@ if [[ $skip_low_pri_tests == 1 ]]; then echo "Skipping some non-critical tests because SKIP_LOW_PRI_TESTS is set." fi -output_dir="/tmp/output" +output_dir="${TMPDIR:-/tmp}/output" ARGS="\ OUTPUT_DIR=$output_dir \ diff --git a/tools/run_leveldb.sh b/tools/run_leveldb.sh index 222401318..884312e3d 100755 --- a/tools/run_leveldb.sh +++ b/tools/run_leveldb.sh @@ -71,7 +71,7 @@ db_dir=${DATA_DIR:-"/tmp/rocksdb/"} do_setup=${DO_SETUP:-1} save_setup=${SAVE_SETUP:-0} -output_dir="/tmp/output" +output_dir="${TMPDIR:-/tmp}/output" ARGS="\ OUTPUT_DIR=$output_dir \ diff --git a/util/bloom_test.cc b/util/bloom_test.cc index 6cc256316..b5b1bd820 100644 --- a/util/bloom_test.cc +++ b/util/bloom_test.cc @@ -33,7 +33,9 @@ namespace rocksdb { static const int kVerbose = 1; static Slice Key(int i, char* buffer) { - memcpy(buffer, &i, sizeof(i)); + std::string s; + PutFixed32(&s, static_cast(i)); + memcpy(buffer, s.c_str(), sizeof(i)); return Slice(buffer, sizeof(i)); } diff --git a/util/coding.h b/util/coding.h index 91f8bbebc..b97e4f5ee 100644 --- a/util/coding.h +++ b/util/coding.h @@ -21,6 +21,11 @@ #include "rocksdb/write_batch.h" #include "port/port.h" +// Some processors does not allow unaligned access to memory +#if defined(__sparc) + #define PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED +#endif + namespace rocksdb { // The maximum length of a varint in bytes for 64-bit. @@ -127,52 +132,52 @@ inline const char* GetVarint32Ptr(const char* p, // -- Implementation of the functions declared above inline void EncodeFixed32(char* buf, uint32_t value) { -#if __BYTE_ORDER == __LITTLE_ENDIAN - memcpy(buf, &value, sizeof(value)); -#else - buf[0] = value & 0xff; - buf[1] = (value >> 8) & 0xff; - buf[2] = (value >> 16) & 0xff; - buf[3] = (value >> 24) & 0xff; -#endif + if (port::kLittleEndian) { + memcpy(buf, &value, sizeof(value)); + } else { + buf[0] = value & 0xff; + buf[1] = (value >> 8) & 0xff; + buf[2] = (value >> 16) & 0xff; + buf[3] = (value >> 24) & 0xff; + } } inline void EncodeFixed64(char* buf, uint64_t value) { -#if __BYTE_ORDER == __LITTLE_ENDIAN - memcpy(buf, &value, sizeof(value)); -#else - buf[0] = value & 0xff; - buf[1] = (value >> 8) & 0xff; - buf[2] = (value >> 16) & 0xff; - buf[3] = (value >> 24) & 0xff; - buf[4] = (value >> 32) & 0xff; - buf[5] = (value >> 40) & 0xff; - buf[6] = (value >> 48) & 0xff; - buf[7] = (value >> 56) & 0xff; -#endif + if (port::kLittleEndian) { + memcpy(buf, &value, sizeof(value)); + } else { + buf[0] = value & 0xff; + buf[1] = (value >> 8) & 0xff; + buf[2] = (value >> 16) & 0xff; + buf[3] = (value >> 24) & 0xff; + buf[4] = (value >> 32) & 0xff; + buf[5] = (value >> 40) & 0xff; + buf[6] = (value >> 48) & 0xff; + buf[7] = (value >> 56) & 0xff; + } } // Pull the last 8 bits and cast it to a character inline void PutFixed32(std::string* dst, uint32_t value) { -#if __BYTE_ORDER__ == __LITTLE_ENDIAN__ - dst->append(const_cast(reinterpret_cast(&value)), - sizeof(value)); -#else - char buf[sizeof(value)]; - EncodeFixed32(buf, value); - dst->append(buf, sizeof(buf)); -#endif + if (port::kLittleEndian) { + dst->append(const_cast(reinterpret_cast(&value)), + sizeof(value)); + } else { + char buf[sizeof(value)]; + EncodeFixed32(buf, value); + dst->append(buf, sizeof(buf)); + } } inline void PutFixed64(std::string* dst, uint64_t value) { -#if __BYTE_ORDER__ == __LITTLE_ENDIAN__ - dst->append(const_cast(reinterpret_cast(&value)), - sizeof(value)); -#else - char buf[sizeof(value)]; - EncodeFixed64(buf, value); - dst->append(buf, sizeof(buf)); -#endif + if (port::kLittleEndian) { + dst->append(const_cast(reinterpret_cast(&value)), + sizeof(value)); + } else { + char buf[sizeof(value)]; + EncodeFixed64(buf, value); + dst->append(buf, sizeof(buf)); + } } inline void PutVarint32(std::string* dst, uint32_t v) { @@ -346,4 +351,24 @@ inline Slice GetSliceUntil(Slice* slice, char delimiter) { return ret; } +template +inline void PutUnaligned(T *memory, const T &value) { +#if defined(PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED) + char *nonAlignedMemory = reinterpret_cast(memory); + memcpy(nonAlignedMemory, reinterpret_cast(&value), sizeof(T)); +#else + *memory = value; +#endif +} + +template +inline void GetUnaligned(const T *memory, T *value) { +#if defined(PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED) + char *nonAlignedMemory = reinterpret_cast(value); + memcpy(nonAlignedMemory, reinterpret_cast(memory), sizeof(T)); +#else + *value = *memory; +#endif +} + } // namespace rocksdb diff --git a/util/delete_scheduler_test.cc b/util/delete_scheduler_test.cc index ad1e3949c..5825374c5 100644 --- a/util/delete_scheduler_test.cc +++ b/util/delete_scheduler_test.cc @@ -91,7 +91,7 @@ TEST_F(DeleteSchedulerTest, BasicRateLimiting) { std::vector penalties; rocksdb::SyncPoint::GetInstance()->SetCallBack( "DeleteScheduler::BackgroundEmptyTrash:Wait", - [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); + [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); int num_files = 100; // 100 files uint64_t file_size = 1024; // every file is 1 kb @@ -158,7 +158,7 @@ TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) { std::vector penalties; rocksdb::SyncPoint::GetInstance()->SetCallBack( "DeleteScheduler::BackgroundEmptyTrash:Wait", - [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); + [&](void* arg) { penalties.push_back(*(static_cast(arg))); }); int thread_cnt = 10; int num_files = 10; // 10 files per thread diff --git a/util/testharness.h b/util/testharness.h index 298b16632..46b4a6cb7 100644 --- a/util/testharness.h +++ b/util/testharness.h @@ -9,7 +9,11 @@ #pragma once +#ifdef OS_AIX +#include "gtest/gtest.h" +#else #include +#endif #include #include "rocksdb/env.h" diff --git a/util/testutil.cc b/util/testutil.cc index 230f9d73c..6d9a5f62e 100644 --- a/util/testutil.cc +++ b/util/testutil.cc @@ -94,9 +94,13 @@ class Uint64ComparatorImpl : public Comparator { assert(a.size() == sizeof(uint64_t) && b.size() == sizeof(uint64_t)); const uint64_t* left = reinterpret_cast(a.data()); const uint64_t* right = reinterpret_cast(b.data()); - if (*left == *right) { + uint64_t leftValue; + uint64_t rightValue; + GetUnaligned(left, &leftValue); + GetUnaligned(right, &rightValue); + if (leftValue == rightValue) { return 0; - } else if (*left < *right) { + } else if (leftValue < rightValue) { return -1; } else { return 1; diff --git a/utilities/column_aware_encoding_test.cc b/utilities/column_aware_encoding_test.cc index d1b5af81e..86d6f94df 100644 --- a/utilities/column_aware_encoding_test.cc +++ b/utilities/column_aware_encoding_test.cc @@ -59,9 +59,9 @@ TEST_P(ColumnAwareEncodingTestWithSize, NoCompressionEncodeDecode) { const char* encoded_data_ptr = encoded_data.c_str(); uint64_t expected_encoded_val; if (col_size == 8) { - expected_encoded_val = 0x0807060504030201; + expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708; } else if (col_size == 4) { - expected_encoded_val = 0x08070605; + expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000; } uint64_t encoded_val = 0; for (int i = 0; i < row_count; ++i) { @@ -122,9 +122,9 @@ TEST_P(ColumnAwareEncodingTestWithSize, RleEncodeDecode) { memcpy(&encoded_val, encoded_data_ptr, col_size); uint64_t expected_encoded_val; if (col_size == 8) { - expected_encoded_val = 0x0807060504030201; + expected_encoded_val = port::kLittleEndian ? 0x0807060504030201 : 0x0102030405060708; } else if (col_size == 4) { - expected_encoded_val = 0x08070605; + expected_encoded_val = port::kLittleEndian ? 0x08070605 : 0x0102030400000000; } // Check correctness of encoded value ASSERT_EQ(expected_encoded_val, encoded_val); @@ -158,8 +158,8 @@ TEST_P(ColumnAwareEncodingTestWithSize, DeltaEncodeDecode) { std::unique_ptr col_buf_encoder( new FixedLengthColBufEncoder(col_size, kColDeltaVarint, false, true)); std::string str_buf; - uint64_t base_val1 = 0x0102030405060708; - uint64_t base_val2 = 0x0202030405060708; + uint64_t base_val1 = port::kLittleEndian ? 0x0102030405060708 : 0x0807060504030201; + uint64_t base_val2 = port::kLittleEndian ? 0x0202030405060708 : 0x0807060504030202; uint64_t val1 = 0, val2 = 0; memcpy(&val1, &base_val1, col_size); memcpy(&val2, &base_val2, col_size); @@ -180,7 +180,7 @@ TEST_P(ColumnAwareEncodingTestWithSize, DeltaEncodeDecode) { if (col_size == 8) { varint_len = 9; } else if (col_size == 4) { - varint_len = 5; + varint_len = port::kLittleEndian ? 5 : 9; } // Check encoded string length: first value is original one (val - 0), the // coming three are encoded as 1, -1, 1, so they should take 1 byte in varint. diff --git a/utilities/persistent_cache/hash_table_test.cc b/utilities/persistent_cache/hash_table_test.cc index eb050604b..e53e72736 100644 --- a/utilities/persistent_cache/hash_table_test.cc +++ b/utilities/persistent_cache/hash_table_test.cc @@ -89,7 +89,7 @@ TEST_F(HashTableTest, TestInsert) { // verify for (uint64_t k = 0; k < max_keys; ++k) { Node val; - port::RWMutex* rlock; + port::RWMutex* rlock = nullptr; assert(map_.Find(Node(k), &val, &rlock)); rlock->ReadUnlock(); assert(val.val_ == std::string(1000, k % 255)); diff --git a/utilities/persistent_cache/persistent_cache_bench.cc b/utilities/persistent_cache/persistent_cache_bench.cc index 112b26555..c57aa0c6b 100644 --- a/utilities/persistent_cache/persistent_cache_bench.cc +++ b/utilities/persistent_cache/persistent_cache_bench.cc @@ -4,6 +4,11 @@ // of patent rights can be found in the PATENTS file in the same directory. // #ifndef ROCKSDB_LITE + +#ifndef GFLAGS +#include +int main() { fprintf(stderr, "Please install gflags to run tools\n"); } +#else #include #include #include @@ -349,6 +354,7 @@ int main(int argc, char** argv) { return 0; } +#endif // #ifndef GFLAGS #else int main(int, char**) { return 0; } #endif