From 7f1451957743a0ced6ded5a1afefc9cb363cb835 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Tue, 26 Nov 2019 10:52:04 -0800 Subject: [PATCH] Small improvements to Docker build for RocksJava (#6079) Summary: * We can reuse downloaded 3rd-party libraries * We can isolate the build to a Docker volume. This is useful for investigating failed builds, as we can examine the volume by assigning it a name during the build. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6079 Differential Revision: D18710263 fbshipit-source-id: 93f456ba44b49e48941c43b0c4d53995ecc1f404 --- Makefile | 25 ++++++++++++-------- java/crossbuild/build-linux-centos.sh | 2 +- java/crossbuild/docker-build-linux-centos.sh | 21 +++++++++------- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 001c87d09..d1f4f6470 100644 --- a/Makefile +++ b/Makefile @@ -1103,14 +1103,23 @@ unity_test: db/db_test.o db/db_test_util.o $(TESTHARNESS) $(TOOLLIBOBJECTS) unit rocksdb.h rocksdb.cc: build_tools/amalgamate.py Makefile $(LIB_SOURCES) unity.cc build_tools/amalgamate.py -I. -i./include unity.cc -x include/rocksdb/c.h -H rocksdb.h -o rocksdb.cc -clean: +clean: clean-ext-libraries-all clean-rocks + +clean-not-downloaded: clean-ext-libraries-bin clean-rocks + +clean-rocks: rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(LIBRARY) $(SHARED) rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report $(FIND) . -name "*.[oda]" -exec rm -f {} \; $(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \; - rm -rf bzip2* snappy* zlib* lz4* zstd* cd java; $(MAKE) clean +clean-ext-libraries-all: + rm -rf bzip2* snappy* zlib* lz4* zstd* + +clean-ext-libraries-bin: + find . -maxdepth 1 -type d \( -name bzip2\* -or -name snappy\* -or -name zlib\* -or -name lz4\* -or -name zstd\* \) -prune -exec rm -rf {} \; + tags: ctags -R . cscope -b `$(FIND) . -name '*.cc'` `$(FIND) . -name '*.h'` `$(FIND) . -name '*.c'` @@ -1913,23 +1922,19 @@ rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 roc rocksdbjavastaticdockerx86: mkdir -p java/target - docker run --rm --name rocksdb_linux_x86-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh + docker run --rm --name rocksdb_linux_x86-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh rocksdbjavastaticdockerx86_64: mkdir -p java/target - docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh + docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh rocksdbjavastaticdockerppc64le: mkdir -p java/target - docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh + docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh rocksdbjavastaticdockerarm64v8: mkdir -p java/target - DOCKER_LINUX_ARM64V8_CONTAINER=`docker ps -aqf name=rocksdb_linux_arm64v8-be`; \ - if [ -z "$$DOCKER_LINUX_ARM64V8_CONTAINER" ]; then \ - docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_arm64v8-be evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \ - fi - docker start -a rocksdb_linux_arm64v8-be + docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral diff --git a/java/crossbuild/build-linux-centos.sh b/java/crossbuild/build-linux-centos.sh index f7090856d..176e3456c 100755 --- a/java/crossbuild/build-linux-centos.sh +++ b/java/crossbuild/build-linux-centos.sh @@ -32,7 +32,7 @@ export PATH=$JAVA_HOME:/usr/local/bin:$PATH # build rocksdb cd /rocksdb -scl enable devtoolset-2 'make jclean clean' +scl enable devtoolset-2 'make clean-not-downloaded' scl enable devtoolset-2 'PORTABLE=1 make -j8 rocksdbjavastatic' cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build diff --git a/java/crossbuild/docker-build-linux-centos.sh b/java/crossbuild/docker-build-linux-centos.sh index 68ee610cb..33fcff955 100755 --- a/java/crossbuild/docker-build-linux-centos.sh +++ b/java/crossbuild/docker-build-linux-centos.sh @@ -4,26 +4,31 @@ set -e #set -x -rm -rf /rocksdb-local -cp -r /rocksdb-host /rocksdb-local -cd /rocksdb-local +# just in-case this is run outside Docker +mkdir -p /rocksdb-local-build -# Use scl devtoolset if available (i.e. CentOS <7) +rm -rf /rocksdb-local-build/* +cp -r /rocksdb-host/* /rocksdb-local-build +cd /rocksdb-local-build + +# Use scl devtoolset if available if hash scl 2>/dev/null; then if scl --list | grep -q 'devtoolset-7'; then - scl enable devtoolset-7 'make jclean clean' + # CentOS 7+ + scl enable devtoolset-7 'make clean-not-downloaded' scl enable devtoolset-7 'PORTABLE=1 make -j2 rocksdbjavastatic' elif scl --list | grep -q 'devtoolset-2'; then - scl enable devtoolset-2 'make jclean clean' + # CentOS 5 or 6 + scl enable devtoolset-2 'make clean-not-downloaded' scl enable devtoolset-2 'PORTABLE=1 make -j2 rocksdbjavastatic' else echo "Could not find devtoolset" exit 1; fi else - make jclean clean + make clean-not-downloaded PORTABLE=1 make -j2 rocksdbjavastatic fi -cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar /rocksdb-host/java/target +cp java/target/librocksdbjni-linux*.so java/target/rocksdbjni-*-linux*.jar /rocksdb-java-target