From dab19afe564801a31024461fa7a8e4251b857f23 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Mon, 7 Mar 2022 10:50:52 -0800 Subject: [PATCH] Fix RocksJava releases for macOS (#9662) Summary: Addresses the problems described in https://github.com/facebook/rocksdb/pull/9254#issuecomment-1054598516 and https://github.com/facebook/rocksdb/pull/9254#issuecomment-1059574837 that have blocked a RocksJava release **NOTE** Also needs to be ported to 6.29.fb branch. Pull Request resolved: https://github.com/facebook/rocksdb/pull/9662 Reviewed By: ajkr Differential Revision: D34689200 Pulled By: pdillinger fbshipit-source-id: c62fe34c54f05be5a00ee1daec8ec7454baa5eb8 --- Makefile | 37 ++++-- .../java/org/rocksdb/NativeLibraryLoader.java | 112 ++++++++++++------ .../java/org/rocksdb/util/Environment.java | 25 +++- .../org/rocksdb/util/EnvironmentTest.java | 29 +++-- 4 files changed, 146 insertions(+), 57 deletions(-) diff --git a/Makefile b/Makefile index 6fd7339a6..7cd92595f 100644 --- a/Makefile +++ b/Makefile @@ -329,8 +329,8 @@ ifneq ($(MACHINE), arm64) # linking with jemalloc (as it won't be arm64-compatible) and remove some other options # set during platform detection DISABLE_JEMALLOC=1 -PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CFLAGS)) -PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CXXFLAGS)) +PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CFLAGS)) +PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CXXFLAGS)) endif endif endif @@ -2121,7 +2121,9 @@ CURL_SSL_OPTS ?= --tlsv1 ifeq ($(PLATFORM), OS_MACOSX) ifeq (,$(findstring librocksdbjni-osx,$(ROCKSDBJNILIB))) ifeq ($(MACHINE),arm64) - ROCKSDBJNILIB = librocksdbjni-osx-aarch64.jnilib + ROCKSDBJNILIB = librocksdbjni-osx-arm64.jnilib +else ifeq ($(MACHINE),x86_64) + ROCKSDBJNILIB = librocksdbjni-osx-x86_64.jnilib else ROCKSDBJNILIB = librocksdbjni-osx.jnilib endif @@ -2252,15 +2254,20 @@ endif $(MAKE) rocksdbjavastatic_deps $(MAKE) rocksdbjavastatic_libobjects $(MAKE) rocksdbjavastatic_javalib - $(MAKE) rocksdbjavastatic_jar + $(MAKE) rocksdbjava_jar rocksdbjavastaticosx: rocksdbjavastaticosx_archs - mv java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx.jnilib - mv java/target/librocksdbjni-osx-arm64.jnilib java/target/librocksdbjni-osx-aarch64.jnilib + cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 rocksdbjavastaticosx_ub: rocksdbjavastaticosx_archs - lipo -create -output ./java/target/$(ROCKSDBJNILIB) java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx-arm64.jnilib - $(MAKE) rocksdbjavastatic_jar + cd java/target; lipo -create -output librocksdbjni-osx.jnilib librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib + cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md + cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx.jnilib + cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class + openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 rocksdbjavastaticosx_archs: $(MAKE) rocksdbjavastaticosx_arch_x86_64 @@ -2294,28 +2301,32 @@ rocksdbjavastatic_javalib: strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \ fi -rocksdbjavastatic_jar: +rocksdbjava_jar: cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB) cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class - cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) * - cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1 + +rocksdbjava_javadocs_jar: + cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) * openssl sha1 java/target/$(ROCKSDB_JAVADOCS_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAVADOCS_JAR).sha1 + +rocksdbjava_sources_jar: + cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org openssl sha1 java/target/$(ROCKSDB_SOURCES_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_SOURCES_JAR).sha1 rocksdbjavastatic_deps: $(JAVA_COMPRESSIONS) rocksdbjavastatic_libobjects: $(LIB_OBJECTS) -rocksdbjavastaticrelease: rocksdbjavastaticosx +rocksdbjavastaticrelease: rocksdbjavastaticosx rocksdbjava_javadocs_jar rocksdbjava_sources_jar cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1 -rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl +rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl rocksdbjava_javadocs_jar rocksdbjava_sources_jar cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class diff --git a/java/src/main/java/org/rocksdb/NativeLibraryLoader.java b/java/src/main/java/org/rocksdb/NativeLibraryLoader.java index e47eacb22..b97cf28b9 100644 --- a/java/src/main/java/org/rocksdb/NativeLibraryLoader.java +++ b/java/src/main/java/org/rocksdb/NativeLibraryLoader.java @@ -18,7 +18,11 @@ public class NativeLibraryLoader { private static final String sharedLibraryName = Environment.getSharedLibraryName("rocksdb"); private static final String jniLibraryName = Environment.getJniLibraryName("rocksdb"); + private static final /* @Nullable */ String fallbackJniLibraryName = + Environment.getFallbackJniLibraryName("rocksdb"); private static final String jniLibraryFileName = Environment.getJniLibraryFileName("rocksdb"); + private static final /* @Nullable */ String fallbackJniLibraryFileName = + Environment.getFallbackJniLibraryFileName("rocksdb"); private static final String tempFilePrefix = "librocksdbjni"; private static final String tempFileSuffix = Environment.getJniLibraryExtension(); @@ -49,14 +53,33 @@ public class NativeLibraryLoader { */ public synchronized void loadLibrary(final String tmpDir) throws IOException { try { - System.loadLibrary(sharedLibraryName); - } catch(final UnsatisfiedLinkError ule1) { + // try dynamic library + System.loadLibrary(sharedLibraryName); + return; + } catch (final UnsatisfiedLinkError ule) { + // ignore - try from static library + } + + try { + // try static library + System.loadLibrary(jniLibraryName); + return; + } catch (final UnsatisfiedLinkError ule) { + // ignore - then try static library fallback or from jar + } + + if (fallbackJniLibraryName != null) { try { - System.loadLibrary(jniLibraryName); - } catch(final UnsatisfiedLinkError ule2) { - loadLibraryFromJar(tmpDir); + // try static library fallback + System.loadLibrary(fallbackJniLibraryName); + return; + } catch (final UnsatisfiedLinkError ule) { + // ignore - then try from jar } } + + // try jar + loadLibraryFromJar(tmpDir); } /** @@ -83,43 +106,62 @@ public class NativeLibraryLoader { File loadLibraryFromJarToTemp(final String tmpDir) throws IOException { - final File temp; - if (tmpDir == null || tmpDir.isEmpty()) { - temp = File.createTempFile(tempFilePrefix, tempFileSuffix); - } else { - final File parentDir = new File(tmpDir); - if (!parentDir.exists()) { - throw new RuntimeException( - "Directory: " + parentDir.getAbsolutePath() + " does not exist!"); + InputStream is = null; + try { + // attempt to look up the static library in the jar file + String libraryFileName = jniLibraryFileName; + is = getClass().getClassLoader().getResourceAsStream(libraryFileName); + + if (is == null) { + // is there a fallback we can try + if (fallbackJniLibraryFileName == null) { + throw new RuntimeException(libraryFileName + " was not found inside JAR."); + } + + // attempt to look up the fallback static library in the jar file + libraryFileName = fallbackJniLibraryFileName; + is = getClass().getClassLoader().getResourceAsStream(libraryFileName); + if (is == null) { + throw new RuntimeException(libraryFileName + " was not found inside JAR."); + } } - temp = new File(parentDir, jniLibraryFileName); - if (temp.exists() && !temp.delete()) { - throw new RuntimeException("File: " + temp.getAbsolutePath() - + " already exists and cannot be removed."); + + // create a temporary file to copy the library to + final File temp; + if (tmpDir == null || tmpDir.isEmpty()) { + temp = File.createTempFile(tempFilePrefix, tempFileSuffix); + } else { + final File parentDir = new File(tmpDir); + if (!parentDir.exists()) { + throw new RuntimeException( + "Directory: " + parentDir.getAbsolutePath() + " does not exist!"); + } + temp = new File(parentDir, libraryFileName); + if (temp.exists() && !temp.delete()) { + throw new RuntimeException( + "File: " + temp.getAbsolutePath() + " already exists and cannot be removed."); + } + if (!temp.createNewFile()) { + throw new RuntimeException("File: " + temp.getAbsolutePath() + " could not be created."); + } } - if (!temp.createNewFile()) { - throw new RuntimeException("File: " + temp.getAbsolutePath() - + " could not be created."); + if (!temp.exists()) { + throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist."); + } else { + temp.deleteOnExit(); } - } - if (!temp.exists()) { - throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist."); - } else { - temp.deleteOnExit(); - } + // copy the library from the Jar file to the temp destination + Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); - // attempt to copy the library from the Jar file to the temp destination - try (final InputStream is = getClass().getClassLoader(). - getResourceAsStream(jniLibraryFileName)) { - if (is == null) { - throw new RuntimeException(jniLibraryFileName + " was not found inside JAR."); - } else { - Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); + // return the temporary library file + return temp; + + } finally { + if (is != null) { + is.close(); } } - - return temp; } /** diff --git a/java/src/main/java/org/rocksdb/util/Environment.java b/java/src/main/java/org/rocksdb/util/Environment.java index 46357a38f..5da471f18 100644 --- a/java/src/main/java/org/rocksdb/util/Environment.java +++ b/java/src/main/java/org/rocksdb/util/Environment.java @@ -110,8 +110,14 @@ public class Environment { return String.format("%sjni-linux%s%s", name, arch, getLibcPostfix()); } } else if (isMac()) { - if (isAarch64()) { - return String.format("%sjni-osx-%s", name, ARCH); + if (is64Bit()) { + final String arch; + if (isAarch64()) { + arch = "arm64"; + } else { + arch = "x86_64"; + } + return String.format("%sjni-osx-%s", name, arch); } else { return String.format("%sjni-osx", name); } @@ -131,10 +137,25 @@ public class Environment { throw new UnsupportedOperationException(String.format("Cannot determine JNI library name for ARCH='%s' OS='%s' name='%s'", ARCH, OS, name)); } + public static /*@Nullable*/ String getFallbackJniLibraryName(final String name) { + if (isMac() && is64Bit()) { + return String.format("%sjni-osx", name); + } + return null; + } + public static String getJniLibraryFileName(final String name) { return appendLibOsSuffix("lib" + getJniLibraryName(name), false); } + public static /*@Nullable*/ String getFallbackJniLibraryFileName(final String name) { + final String fallbackJniLibraryName = getFallbackJniLibraryName(name); + if (fallbackJniLibraryName == null) { + return null; + } + return appendLibOsSuffix("lib" + fallbackJniLibraryName, false); + } + private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) { if (isUnix() || isAix() || isSolaris() || isFreeBSD() || isOpenBSD()) { return libraryFileName + ".so"; diff --git a/java/src/test/java/org/rocksdb/util/EnvironmentTest.java b/java/src/test/java/org/rocksdb/util/EnvironmentTest.java index 43cd38a1c..301dec22f 100644 --- a/java/src/test/java/org/rocksdb/util/EnvironmentTest.java +++ b/java/src/test/java/org/rocksdb/util/EnvironmentTest.java @@ -9,7 +9,6 @@ import org.junit.BeforeClass; import org.junit.Test; import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import static org.assertj.core.api.Assertions.assertThat; @@ -37,18 +36,21 @@ public class EnvironmentTest { isEqualTo(".jnilib"); assertThat(Environment.getJniLibraryFileName("rocksdb")). isEqualTo("librocksdbjni-osx.jnilib"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.dylib"); } @Test - public void mac64() { - setEnvironmentClassFields("mac", "x86-64"); + public void mac64_x86_64() { + setEnvironmentClassFields("mac", "x86_64"); assertThat(Environment.isWindows()).isFalse(); assertThat(Environment.getJniLibraryExtension()). isEqualTo(".jnilib"); - assertThat(Environment.getJniLibraryFileName("rocksdb")). - isEqualTo("librocksdbjni-osx.jnilib"); + assertThat(Environment.getJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-osx-x86_64.jnilib"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-osx.jnilib"); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.dylib"); } @@ -59,7 +61,9 @@ public class EnvironmentTest { assertThat(Environment.isWindows()).isFalse(); assertThat(Environment.getJniLibraryExtension()).isEqualTo(".jnilib"); assertThat(Environment.getJniLibraryFileName("rocksdb")) - .isEqualTo("librocksdbjni-osx-aarch64.jnilib"); + .isEqualTo("librocksdbjni-osx-arm64.jnilib"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")) + .isEqualTo("librocksdbjni-osx.jnilib"); assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.dylib"); } @@ -73,6 +77,7 @@ public class EnvironmentTest { isEqualTo(".so"); assertThat(Environment.getJniLibraryFileName("rocksdb")). isEqualTo("librocksdbjni-linux32.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.so"); // Linux musl-libc (Alpine) @@ -103,7 +108,8 @@ public class EnvironmentTest { assertThat(Environment.isWindows()).isFalse(); assertThat(Environment.getJniLibraryExtension()). isEqualTo(".so"); - Environment.getJniLibraryFileName("rocksdb"); + assertThat(Environment.getJniLibraryFileName("rocksdb")).isEqualTo("blah"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); } @Test @@ -115,6 +121,7 @@ public class EnvironmentTest { isEqualTo(".so"); assertThat(Environment.getJniLibraryFileName("rocksdb")). isEqualTo("librocksdbjni-linux64.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.so"); // Linux musl-libc (Alpine) @@ -124,6 +131,7 @@ public class EnvironmentTest { isEqualTo(".so"); assertThat(Environment.getJniLibraryFileName("rocksdb")). isEqualTo("librocksdbjni-linux64-musl.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.so"); // UNIX @@ -134,6 +142,7 @@ public class EnvironmentTest { isEqualTo(".so"); assertThat(Environment.getJniLibraryFileName("rocksdb")). isEqualTo("librocksdbjni-linux64.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.so"); // AIX @@ -143,6 +152,7 @@ public class EnvironmentTest { isEqualTo(".so"); assertThat(Environment.getJniLibraryFileName("rocksdb")). isEqualTo("librocksdbjni-aix64.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.so"); } @@ -161,6 +171,7 @@ public class EnvironmentTest { isEqualTo(".dll"); assertThat(Environment.getJniLibraryFileName("rocksdb")). isEqualTo("librocksdbjni-win64.dll"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")). isEqualTo("librocksdbjni.dll"); } @@ -177,6 +188,7 @@ public class EnvironmentTest { assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-ppc64le"); assertThat(Environment.getJniLibraryFileName("rocksdb")) .isEqualTo("librocksdbjni-linux-ppc64le.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so"); // Linux musl-libc (Alpine) setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true); @@ -189,6 +201,7 @@ public class EnvironmentTest { assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-ppc64le-musl"); assertThat(Environment.getJniLibraryFileName("rocksdb")) .isEqualTo("librocksdbjni-linux-ppc64le-musl.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so"); setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); } @@ -205,6 +218,7 @@ public class EnvironmentTest { assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-aarch64"); assertThat(Environment.getJniLibraryFileName("rocksdb")) .isEqualTo("librocksdbjni-linux-aarch64.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so"); // Linux musl-libc (Alpine) setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true); @@ -217,6 +231,7 @@ public class EnvironmentTest { assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-aarch64-musl"); assertThat(Environment.getJniLibraryFileName("rocksdb")) .isEqualTo("librocksdbjni-linux-aarch64-musl.so"); + assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull(); assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so"); setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false); }