From 0227c16d67d784ca5c2342d008a4e92d3f4bb1ec Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sun, 12 Feb 2017 22:53:42 -0800 Subject: [PATCH] Update static library versions and add checksums Summary: The previous version of zlib is no longer available. I have also updated the versions of the other static libraries and added checkum checks for the downloads; This is related to https://github.com/facebook/rocksdb/issues/1769 Closes https://github.com/facebook/rocksdb/pull/1863 Differential Revision: D4550742 Pulled By: yiwu-arbug fbshipit-source-id: 4414150 --- INSTALL.md | 28 ++++++++++++++------ Makefile | 78 ++++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 75 insertions(+), 31 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 3b2651118..da7921b8f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -29,8 +29,10 @@ your make commands, like this: `PORTABLE=1 make static_lib` * You can link RocksDB with following compression libraries: - [zlib](http://www.zlib.net/) - a library for data compression. - [bzip2](http://www.bzip.org/) - a library for data compression. - - [snappy](https://code.google.com/p/snappy/) - a library for fast + - [snappy](http://google.github.io/snappy/) - a library for fast data compression. + - [zstandard](http://www.zstd.net) - Fast real-time compression + algorithm. * All our tools depend on: - [gflags](https://gflags.github.io/gflags/) - a library that handles @@ -48,21 +50,23 @@ your make commands, like this: `PORTABLE=1 make static_lib` `sudo apt-get install libsnappy-dev`. * Install zlib. Try: `sudo apt-get install zlib1g-dev`. * Install bzip2: `sudo apt-get install libbz2-dev`. -* **Linux - CentOS** + * Install zstandard: `sudo apt-get install libzstd-dev`. + +* **Linux - CentOS / RHEL** * Upgrade your gcc to version at least 4.8 to get C++11 support: `yum install gcc48-c++` * Install gflags: - wget https://gflags.googlecode.com/files/gflags-2.0-no-svn-files.tar.gz - tar -xzvf gflags-2.0-no-svn-files.tar.gz - cd gflags-2.0 + git clone https://github.com/gflags/gflags.git + git checkout v2.0 + cd gflags ./configure && make && sudo make install * Install snappy: - wget https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz - tar -xzvf snappy-1.1.1.tar.gz - cd snappy-1.1.1 + wget https://github.com/google/snappy/releases/download/1.1.4/snappy-1.1.4.tar.gz + tar -xzvf snappy-1.1.4.tar.gz + cd snappy-1.1.4 ./configure && make && sudo make install * Install zlib: @@ -75,6 +79,14 @@ your make commands, like this: `PORTABLE=1 make static_lib` sudo yum install bzip2 sudo yum install bzip2-devel + * Install zstandard: + + wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz + mv v1.1.3.tar.gz zstandard-1.1.3.tar.gz + tar zxvf zstd-1.1.3.tar.gz + cd zstd-1.1.3 + make && sudo make install + * **OS X**: * Install latest C++ compiler that supports C++ 11: * Update XCode: run `xcode-select --install` (or install it from XCode App's settting). diff --git a/Makefile b/Makefile index 69adcaaf4..165e7558d 100644 --- a/Makefile +++ b/Makefile @@ -1370,10 +1370,21 @@ ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linu ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-javadoc.jar ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar +SHA256_CMD = sha256sum + +ZLIB_VER = 1.2.11 +ZLIB_SHA256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 +BZIP2_VER = 1.0.6 +BZIP2_SHA256 = a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd +SNAPPY_VER = 1.1.4 +SNAPPY_SHA256 = 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057 +LZ4_VER = 1.7.5 +LZ4_SHA256 = 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e ifeq ($(PLATFORM), OS_MACOSX) ROCKSDBJNILIB = librocksdbjni-osx.jnilib ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar + SHA256_CMD = openssl sha256 -r ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","") JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin else @@ -1389,44 +1400,65 @@ ifeq ($(PLATFORM), OS_SOLARIS) ROCKSDBJNILIB = librocksdbjni-solaris$(ARCH).so ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-solaris$(ARCH).jar JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris + SHA256_CMD = digest -a sha256 endif libz.a: - -rm -rf zlib-1.2.10 - curl -O -L http://zlib.net/zlib-1.2.10.tar.gz - tar xvzf zlib-1.2.10.tar.gz - cd zlib-1.2.10 && CFLAGS='-fPIC' ./configure --static && make - cp zlib-1.2.10/libz.a . + -rm -rf zlib-$(ZLIB_VER) + curl -O -L http://zlib.net/zlib-$(ZLIB_VER).tar.gz + ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \ + if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \ + echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \ + exit 1; \ + fi + tar xvzf zlib-$(ZLIB_VER).tar.gz + cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC' ./configure --static && make + cp zlib-$(ZLIB_VER)/libz.a . libbz2.a: - -rm -rf bzip2-1.0.6 - curl -O -L http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz - tar xvzf bzip2-1.0.6.tar.gz - cd bzip2-1.0.6 && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64' - cp bzip2-1.0.6/libbz2.a . + -rm -rf bzip2-$(BZIP2_VER) + curl -O -L http://www.bzip.org/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz + BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \ + if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \ + echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \ + exit 1; \ + fi + tar xvzf bzip2-$(BZIP2_VER).tar.gz + cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64' + cp bzip2-$(BZIP2_VER)/libbz2.a . libsnappy.a: - -rm -rf snappy-1.1.3 - curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz - tar xvzf snappy-1.1.3.tar.gz - cd snappy-1.1.3 && ./configure --with-pic --enable-static - cd snappy-1.1.3 && make - cp snappy-1.1.3/.libs/libsnappy.a . + -rm -rf snappy-$(SNAPPY_VER) + curl -O -L https://github.com/google/snappy/releases/download/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz + SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \ + if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \ + echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \ + exit 1; \ + fi + tar xvzf snappy-$(SNAPPY_VER).tar.gz + cd snappy-$(SNAPPY_VER) && ./configure --with-pic --enable-static + cd snappy-$(SNAPPY_VER) && make + cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a . liblz4.a: - -rm -rf lz4-r127 - curl -O -L https://codeload.github.com/Cyan4973/lz4/tar.gz/r127 - mv r127 lz4-r127.tar.gz - tar xvzf lz4-r127.tar.gz - cd lz4-r127/lib && make CFLAGS='-fPIC' all - cp lz4-r127/lib/liblz4.a . + -rm -rf lz4-$(LZ4_VER) + curl -O -L https://github.com/lz4/lz4/archive/v$(LZ4_VER).tar.gz + mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz + LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \ + if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \ + echo lz4-$(LZ4_VER).tar.gz checksum mismatch, expected=\"$(LZ4_SHA256)\" actual=\"$$LZ4_SHA256_ACTUAL\"; \ + exit 1; \ + fi + tar xvzf lz4-$(LZ4_VER).tar.gz + cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC' all + cp lz4-$(LZ4_VER)/lib/liblz4.a . # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS)) CLEAN_FILES += jls JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -JAVA_STATIC_INCLUDES = -I./zlib-1.2.10 -I./bzip2-1.0.6 -I./snappy-1.1.3 -I./lz4-r127/lib +JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib $(java_static_libobjects): jls/%.o: %.cc libz.a libbz2.a libsnappy.a liblz4.a $(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)