Add support for building on s390x platform (#8962)

Summary:
This PR adds support for building on s390x including updating travis CI. It uses the previous work in https://github.com/facebook/rocksdb/pull/6168 and adds some more changes to get all current tests (make check and jni tests) to pass. The tests were run with snappy, lz4, bzip2 and zstd all compiled in.

There are a few pieces still needed to get the travis build working that I don't think I can do. adamretter is this something you could help with?

1. A prebuilt https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/cmake-3.14.5-Linux-s390x.deb package
2. A https://hub.docker.com/r/evolvedbinary/rocksjava s390x image

Not sure if there is more required for travis. Happy to help in any way I can.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8962

Reviewed By: mrambacher

Differential Revision: D31802198

Pulled By: pdillinger

fbshipit-source-id: 683511466fa6b505f85ba5a9964a268c6151f0c2
main
Jonathan Albrecht 3 years ago committed by Facebook GitHub Bot
parent f72fd58565
commit e970248602
  1. 51
      .travis.yml
  2. 14
      CMakeLists.txt
  3. 4
      Makefile
  4. 14
      build_tools/build_detect_platform
  5. 5
      db/db_properties_test.cc
  6. 4
      db/version_set_test.cc
  7. 2
      java/src/main/java/org/rocksdb/util/Environment.java
  8. 2
      monitoring/statistics.h
  9. 4
      port/port_posix.h
  10. 7
      third-party/folly/folly/Portability.h
  11. 2
      third-party/folly/folly/lang/Align.h
  12. 3
      util/bloom_test.cc
  13. 6
      util/compression.h
  14. 4
      utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h

@ -5,6 +5,7 @@ os:
arch: arch:
- arm64 - arm64
- ppc64le - ppc64le
- s390x
compiler: compiler:
- clang - clang
- gcc - gcc
@ -59,6 +60,12 @@ matrix:
- os: linux - os: linux
arch: ppc64le arch: ppc64le
env: JOB_NAME=make-gcc4.8 env: JOB_NAME=make-gcc4.8
- os: linux
arch: s390x
env: JOB_NAME=cmake-mingw
- os: linux
arch: s390x
env: JOB_NAME=make-gcc4.8
- os: linux - os: linux
compiler: clang compiler: clang
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
@ -73,6 +80,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: TEST_GROUP=1 env: TEST_GROUP=1
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: TEST_GROUP=1
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -81,6 +92,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: TEST_GROUP=2 env: TEST_GROUP=2
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: TEST_GROUP=2
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -89,6 +104,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: TEST_GROUP=3 env: TEST_GROUP=3
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: TEST_GROUP=3
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -97,6 +116,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: TEST_GROUP=4 env: TEST_GROUP=4
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: TEST_GROUP=4
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -109,6 +132,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: JOB_NAME=java_test env: JOB_NAME=java_test
- if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
os: linux
arch: s390x
env: JOB_NAME=java_test
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -117,6 +144,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: JOB_NAME=lite_build env: JOB_NAME=lite_build
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=lite_build
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -125,6 +156,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: JOB_NAME=examples env: JOB_NAME=examples
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=examples
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -133,6 +168,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: JOB_NAME=cmake-gcc8 env: JOB_NAME=cmake-gcc8
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=cmake-gcc8
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -141,6 +180,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: JOB_NAME=cmake-gcc9 env: JOB_NAME=cmake-gcc9
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=cmake-gcc9
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -149,6 +192,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: JOB_NAME=cmake-gcc9-c++20 env: JOB_NAME=cmake-gcc9-c++20
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=cmake-gcc9-c++20
- if: type = pull_request AND commit_message !~ /FULL_CI/ - if: type = pull_request AND commit_message !~ /FULL_CI/
os : linux os : linux
arch: arm64 arch: arm64
@ -157,6 +204,10 @@ matrix:
os: linux os: linux
arch: ppc64le arch: ppc64le
env: JOB_NAME=status_checked env: JOB_NAME=status_checked
- if: type = pull_request AND commit_message !~ /FULL_CI/
os: linux
arch: s390x
env: JOB_NAME=status_checked
install: install:
- if [ "${JOB_NAME}" == cmake-gcc8 ]; then - if [ "${JOB_NAME}" == cmake-gcc8 ]; then

@ -267,6 +267,13 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
endif(HAS_ARMV8_CRC) endif(HAS_ARMV8_CRC)
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64") endif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
CHECK_C_COMPILER_FLAG("-march=native" HAS_S390X_MARCH_NATIVE)
if(HAS_S390X_MARCH_NATIVE)
message(STATUS " HAS_S390X_MARCH_NATIVE yes")
endif(HAS_S390X_MARCH_NATIVE)
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
option(PORTABLE "build a portable binary" OFF) option(PORTABLE "build a portable binary" OFF)
option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF) option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
option(FORCE_AVX "force building with AVX, even when PORTABLE=ON" OFF) option(FORCE_AVX "force building with AVX, even when PORTABLE=ON" OFF)
@ -292,12 +299,17 @@ if(PORTABLE)
if(FORCE_AVX2) if(FORCE_AVX2)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2 -mbmi -mlzcnt") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2 -mbmi -mlzcnt")
endif() endif()
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196")
endif()
endif() endif()
else() else()
if(MSVC) if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
else() else()
if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64" AND NOT HAS_ARMV8_CRC) if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x" AND NOT HAS_S390X_MARCH_NATIVE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196")
elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64" AND NOT HAS_ARMV8_CRC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
endif() endif()
endif() endif()

@ -1997,7 +1997,7 @@ JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
ifeq ($(PLATFORM), OS_SOLARIS) ifeq ($(PLATFORM), OS_SOLARIS)
ARCH := $(shell isainfo -b) ARCH := $(shell isainfo -b)
else ifeq ($(PLATFORM), OS_OPENBSD) else ifeq ($(PLATFORM), OS_OPENBSD)
ifneq (,$(filter amd64 ppc64 ppc64le arm64 aarch64 sparc64, $(MACHINE))) ifneq (,$(filter amd64 ppc64 ppc64le s390x arm64 aarch64 sparc64, $(MACHINE)))
ARCH := 64 ARCH := 64
else else
ARCH := 32 ARCH := 32
@ -2017,7 +2017,7 @@ ifneq ($(origin JNI_LIBC), undefined)
JNI_LIBC_POSTFIX = -$(JNI_LIBC) JNI_LIBC_POSTFIX = -$(JNI_LIBC)
endif endif
ifneq (,$(filter ppc% arm64 aarch64 sparc64, $(MACHINE))) ifneq (,$(filter ppc% s390x arm64 aarch64 sparc64, $(MACHINE)))
ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so
else else
ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so

@ -638,13 +638,19 @@ if test "0$PORTABLE" -eq 0; then
# Tune for this POWER processor, treating '+' models as base models # Tune for this POWER processor, treating '+' models as base models
POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+` POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER " COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
elif test -n "`echo $TARGET_ARCHITECTURE | grep -e^arm -e^aarch64`"; then elif test -n "`echo $TARGET_ARCHITECTURE | grep -e^arm -e^aarch64`"; then
# TODO: Handle this with approprite options. # TODO: Handle this with approprite options.
COMMON_FLAGS="$COMMON_FLAGS" COMMON_FLAGS="$COMMON_FLAGS"
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^aarch64`"; then elif test -n "`echo $TARGET_ARCHITECTURE | grep ^aarch64`"; then
COMMON_FLAGS="$COMMON_FLAGS" COMMON_FLAGS="$COMMON_FLAGS"
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ \
-fsyntax-only -march=native - -o /dev/null 2>/dev/null; then
COMMON_FLAGS="$COMMON_FLAGS -march=native "
else
COMMON_FLAGS="$COMMON_FLAGS -march=z196 "
fi
COMMON_FLAGS="$COMMON_FLAGS"
elif [ "$TARGET_OS" == "IOS" ]; then elif [ "$TARGET_OS" == "IOS" ]; then
COMMON_FLAGS="$COMMON_FLAGS" COMMON_FLAGS="$COMMON_FLAGS"
elif [ "$TARGET_OS" == "AIX" ] || [ "$TARGET_OS" == "SunOS" ]; then elif [ "$TARGET_OS" == "AIX" ] || [ "$TARGET_OS" == "SunOS" ]; then
@ -661,6 +667,10 @@ else
TRY_SSE_ETC="1" TRY_SSE_ETC="1"
fi fi
if test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
COMMON_FLAGS="$COMMON_FLAGS -march=z196 "
fi
if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then
# For portability compile for macOS 10.12 (2016) or newer # For portability compile for macOS 10.12 (2016) or newer
COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.12" COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.12"

@ -233,7 +233,7 @@ void VerifySimilar(uint64_t a, uint64_t b, double bias) {
void VerifyTableProperties( void VerifyTableProperties(
const TableProperties& base_tp, const TableProperties& new_tp, const TableProperties& base_tp, const TableProperties& new_tp,
double filter_size_bias = CACHE_LINE_SIZE >= 256 ? 0.15 : 0.1, double filter_size_bias = CACHE_LINE_SIZE >= 256 ? 0.18 : 0.1,
double index_size_bias = 0.1, double data_size_bias = 0.1, double index_size_bias = 0.1, double data_size_bias = 0.1,
double num_data_blocks_bias = 0.05) { double num_data_blocks_bias = 0.05) {
VerifySimilar(base_tp.data_size, new_tp.data_size, data_size_bias); VerifySimilar(base_tp.data_size, new_tp.data_size, data_size_bias);
@ -627,7 +627,8 @@ TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) {
value_is_delta_encoded); value_is_delta_encoded);
// Gives larger bias here as index block size, filter block size, // Gives larger bias here as index block size, filter block size,
// and data block size become much harder to estimate in this test. // and data block size become much harder to estimate in this test.
VerifyTableProperties(expected_tp, tp, 0.5, 0.4, 0.4, 0.25); VerifyTableProperties(expected_tp, tp, CACHE_LINE_SIZE >= 256 ? 0.6 : 0.5,
0.4, 0.4, 0.25);
} }
} }
} }

@ -2083,7 +2083,7 @@ class VersionSetAtomicGroupTest : public VersionSetTestBase,
}); });
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"VersionEditHandlerBase::Iterate:Finish", [&](void* arg) { "VersionEditHandlerBase::Iterate:Finish", [&](void* arg) {
num_recovered_edits_ = *reinterpret_cast<int*>(arg); num_recovered_edits_ = *reinterpret_cast<size_t*>(arg);
}); });
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"AtomicGroupReadBuffer::AddEdit:AtomicGroup", "AtomicGroupReadBuffer::AddEdit:AtomicGroup",
@ -2123,7 +2123,7 @@ class VersionSetAtomicGroupTest : public VersionSetTestBase,
bool first_in_atomic_group_ = false; bool first_in_atomic_group_ = false;
bool last_in_atomic_group_ = false; bool last_in_atomic_group_ = false;
int num_edits_in_atomic_group_ = 0; int num_edits_in_atomic_group_ = 0;
int num_recovered_edits_ = 0; size_t num_recovered_edits_ = 0;
VersionEdit corrupted_edit_; VersionEdit corrupted_edit_;
VersionEdit edit_with_incorrect_group_size_; VersionEdit edit_with_incorrect_group_size_;
std::unique_ptr<log::Writer> log_writer_; std::unique_ptr<log::Writer> log_writer_;

@ -106,7 +106,7 @@ public class Environment {
if (isPowerPC() || isAarch64()) { if (isPowerPC() || isAarch64()) {
return String.format("%sjni-linux-%s%s", name, ARCH, getLibcPostfix()); return String.format("%sjni-linux-%s%s", name, ARCH, getLibcPostfix());
} else if (isS390x()) { } else if (isS390x()) {
return String.format("%sjni-linux%s", name, ARCH); return String.format("%sjni-linux-%s", name, ARCH);
} else { } else {
return String.format("%sjni-linux%s%s", name, arch, getLibcPostfix()); return String.format("%sjni-linux%s%s", name, arch, getLibcPostfix());
} }

@ -100,7 +100,9 @@ class StatisticsImpl : public Statistics {
void operator delete[](void *p) { port::cacheline_aligned_free(p); } void operator delete[](void *p) { port::cacheline_aligned_free(p); }
}; };
#ifndef TEST_CACHE_LINE_SIZE
static_assert(sizeof(StatisticsData) % CACHE_LINE_SIZE == 0, "Expected " TOSTRING(CACHE_LINE_SIZE) "-byte aligned"); static_assert(sizeof(StatisticsData) % CACHE_LINE_SIZE == 0, "Expected " TOSTRING(CACHE_LINE_SIZE) "-byte aligned");
#endif
CoreLocalArray<StatisticsData> per_core_stats_; CoreLocalArray<StatisticsData> per_core_stats_;

@ -193,7 +193,11 @@ extern void InitOnce(OnceType* once, void (*initializer)());
#define ALIGN_AS(n) /*empty*/ #define ALIGN_AS(n) /*empty*/
#else #else
#if defined(__s390__) #if defined(__s390__)
#if defined(__GNUC__) && __GNUC__ < 6
#define CACHE_LINE_SIZE 64U
#else
#define CACHE_LINE_SIZE 256U #define CACHE_LINE_SIZE 256U
#endif
#elif defined(__powerpc__) || defined(__aarch64__) #elif defined(__powerpc__) || defined(__aarch64__)
#define CACHE_LINE_SIZE 128U #define CACHE_LINE_SIZE 128U
#else #else

@ -31,6 +31,12 @@
#define FOLLY_PPC64 0 #define FOLLY_PPC64 0
#endif #endif
#if defined(__s390x__)
#define FOLLY_S390X 1
#else
#define FOLLY_S390X 0
#endif
#if defined(__has_builtin) #if defined(__has_builtin)
#define FOLLY_HAS_BUILTIN(...) __has_builtin(__VA_ARGS__) #define FOLLY_HAS_BUILTIN(...) __has_builtin(__VA_ARGS__)
#else #else
@ -57,6 +63,7 @@ constexpr bool kIsArchArm = FOLLY_ARM == 1;
constexpr bool kIsArchAmd64 = FOLLY_X64 == 1; constexpr bool kIsArchAmd64 = FOLLY_X64 == 1;
constexpr bool kIsArchAArch64 = FOLLY_AARCH64 == 1; constexpr bool kIsArchAArch64 = FOLLY_AARCH64 == 1;
constexpr bool kIsArchPPC64 = FOLLY_PPC64 == 1; constexpr bool kIsArchPPC64 = FOLLY_PPC64 == 1;
constexpr bool kIsArchS390X = FOLLY_S390X == 1;
} // namespace folly } // namespace folly
namespace folly { namespace folly {

@ -120,7 +120,7 @@ struct alignas(max_align_v) max_align_t {};
// //
// mimic: std::hardware_destructive_interference_size, C++17 // mimic: std::hardware_destructive_interference_size, C++17
constexpr std::size_t hardware_destructive_interference_size = constexpr std::size_t hardware_destructive_interference_size =
kIsArchArm ? 64 : 128; (kIsArchArm || kIsArchS390X) ? 64 : 128;
static_assert(hardware_destructive_interference_size >= max_align_v, "math?"); static_assert(hardware_destructive_interference_size >= max_align_v, "math?");
// Memory locations within the same cache line are subject to constructive // Memory locations within the same cache line are subject to constructive

@ -552,7 +552,8 @@ TEST_P(FullBloomTest, OptimizeForMemory) {
} }
if (FLAGS_bits_per_key == 10) { if (FLAGS_bits_per_key == 10) {
EXPECT_LE(total_fp_rate / double{nfilters}, 0.011); EXPECT_LE(total_fp_rate / double{nfilters}, 0.011);
EXPECT_GE(total_fp_rate / double{nfilters}, 0.008); EXPECT_GE(total_fp_rate / double{nfilters},
CACHE_LINE_SIZE >= 256 ? 0.007 : 0.008);
} }
int64_t ex_min_total_size = int64_t{FLAGS_bits_per_key} * total_keys / 8; int64_t ex_min_total_size = int64_t{FLAGS_bits_per_key} * total_keys / 8;

@ -1136,7 +1136,11 @@ inline CacheAllocationPtr LZ4_Uncompress(const UncompressionInfo& info,
if (input_length < 8) { if (input_length < 8) {
return nullptr; return nullptr;
} }
memcpy(&output_len, input_data, sizeof(output_len)); if (port::kLittleEndian) {
memcpy(&output_len, input_data, sizeof(output_len));
} else {
memcpy(&output_len, input_data + 4, sizeof(output_len));
}
input_length -= 8; input_length -= 8;
input_data += 8; input_data += 8;
} }

@ -133,6 +133,10 @@ static inline tokutime_t toku_time_now(void) {
return result; return result;
#elif defined(__powerpc__) #elif defined(__powerpc__)
return __ppc_get_timebase(); return __ppc_get_timebase();
#elif defined(__s390x__)
uint64_t result;
asm volatile("stckf %0" : "=Q"(result) : : "cc");
return result;
#else #else
#error No timer implementation for this platform #error No timer implementation for this platform
#endif #endif

Loading…
Cancel
Save