NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix RocksJava releases for macOS (#9662)

Browse Source 
Summary:
Addresses the problems described in https://github.com/facebook/rocksdb/pull/9254#issuecomment-1054598516 and https://github.com/facebook/rocksdb/pull/9254#issuecomment-1059574837 that have blocked a RocksJava release

**NOTE** Also needs to be ported to 6.29.fb branch.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9662

Reviewed By: ajkr

Differential Revision: D34689200

Pulled By: pdillinger

fbshipit-source-id: c62fe34c54f05be5a00ee1daec8ec7454baa5eb8
main
Adam Retter 2 years ago committed by Facebook GitHub Bot
parent f20b674796
commit dab19afe56
4 changed files with 146 additions and 57 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 37
      Makefile
  2. 112
      java/src/main/java/org/rocksdb/NativeLibraryLoader.java
  3. 25
      java/src/main/java/org/rocksdb/util/Environment.java
  4. 29
      java/src/test/java/org/rocksdb/util/EnvironmentTest.java

37
Makefile
Unescape View File

@ -329,8 +329,8 @@ ifneq ($(MACHINE), arm64)
# linking with jemalloc (as it won't be arm64-compatible) and remove some other options
# set during platform detection
DISABLE_JEMALLOC=1
PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CFLAGS))
PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CXXFLAGS))
PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CFLAGS))
PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CXXFLAGS))
endif
endif
endif
@ -2121,7 +2121,9 @@ CURL_SSL_OPTS ?= --tlsv1
ifeq ($(PLATFORM), OS_MACOSX)
ifeq (,$(findstring librocksdbjni-osx,$(ROCKSDBJNILIB)))
ifeq ($(MACHINE),arm64)
ROCKSDBJNILIB = librocksdbjni-osx-aarch64.jnilib
ROCKSDBJNILIB = librocksdbjni-osx-arm64.jnilib
else ifeq ($(MACHINE),x86_64)
ROCKSDBJNILIB = librocksdbjni-osx-x86_64.jnilib
else
ROCKSDBJNILIB = librocksdbjni-osx.jnilib
endif
@ -2252,15 +2254,20 @@ endif
$(MAKE) rocksdbjavastatic_deps
$(MAKE) rocksdbjavastatic_libobjects
$(MAKE) rocksdbjavastatic_javalib
$(MAKE) rocksdbjavastatic_jar
$(MAKE) rocksdbjava_jar
rocksdbjavastaticosx: rocksdbjavastaticosx_archs
mv java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx.jnilib
mv java/target/librocksdbjni-osx-arm64.jnilib java/target/librocksdbjni-osx-aarch64.jnilib
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
rocksdbjavastaticosx_ub: rocksdbjavastaticosx_archs
lipo -create -output ./java/target/$(ROCKSDBJNILIB) java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx-arm64.jnilib
$(MAKE) rocksdbjavastatic_jar
cd java/target; lipo -create -output librocksdbjni-osx.jnilib librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
rocksdbjavastaticosx_archs:
$(MAKE) rocksdbjavastaticosx_arch_x86_64
@ -2294,28 +2301,32 @@ rocksdbjavastatic_javalib:
strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
fi
rocksdbjavastatic_jar:
rocksdbjava_jar:
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) *
cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
rocksdbjava_javadocs_jar:
cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) *
openssl sha1 java/target/$(ROCKSDB_JAVADOCS_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAVADOCS_JAR).sha1
rocksdbjava_sources_jar:
cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
openssl sha1 java/target/$(ROCKSDB_SOURCES_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_SOURCES_JAR).sha1
rocksdbjavastatic_deps: $(JAVA_COMPRESSIONS)
rocksdbjavastatic_libobjects: $(LIB_OBJECTS)
rocksdbjavastaticrelease: rocksdbjavastaticosx
rocksdbjavastaticrelease: rocksdbjavastaticosx rocksdbjava_javadocs_jar rocksdbjava_sources_jar
cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1
rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl rocksdbjava_javadocs_jar rocksdbjava_sources_jar
cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class

112
java/src/main/java/org/rocksdb/NativeLibraryLoader.java
Unescape View File

@ -18,7 +18,11 @@ public class NativeLibraryLoader {
private static final String sharedLibraryName = Environment.getSharedLibraryName("rocksdb");
private static final String jniLibraryName = Environment.getJniLibraryName("rocksdb");
private static final /* @Nullable */ String fallbackJniLibraryName =
Environment.getFallbackJniLibraryName("rocksdb");
private static final String jniLibraryFileName = Environment.getJniLibraryFileName("rocksdb");
private static final /* @Nullable */ String fallbackJniLibraryFileName =
Environment.getFallbackJniLibraryFileName("rocksdb");
private static final String tempFilePrefix = "librocksdbjni";
private static final String tempFileSuffix = Environment.getJniLibraryExtension();
@ -49,14 +53,33 @@ public class NativeLibraryLoader {
*/
public synchronized void loadLibrary(final String tmpDir) throws IOException {
try {
System.loadLibrary(sharedLibraryName);
} catch(final UnsatisfiedLinkError ule1) {
// try dynamic library
System.loadLibrary(sharedLibraryName);
return;
} catch (final UnsatisfiedLinkError ule) {
// ignore - try from static library
}
try {
// try static library
System.loadLibrary(jniLibraryName);
return;
} catch (final UnsatisfiedLinkError ule) {
// ignore - then try static library fallback or from jar
}
if (fallbackJniLibraryName != null) {
try {
System.loadLibrary(jniLibraryName);
} catch(final UnsatisfiedLinkError ule2) {
loadLibraryFromJar(tmpDir);
// try static library fallback
System.loadLibrary(fallbackJniLibraryName);
return;
} catch (final UnsatisfiedLinkError ule) {
// ignore - then try from jar
}
}
// try jar
loadLibraryFromJar(tmpDir);
}
/**
@ -83,43 +106,62 @@ public class NativeLibraryLoader {
File loadLibraryFromJarToTemp(final String tmpDir)
throws IOException {
final File temp;
if (tmpDir == null || tmpDir.isEmpty()) {
temp = File.createTempFile(tempFilePrefix, tempFileSuffix);
} else {
final File parentDir = new File(tmpDir);
if (!parentDir.exists()) {
throw new RuntimeException(
"Directory: " + parentDir.getAbsolutePath() + " does not exist!");
InputStream is = null;
try {
// attempt to look up the static library in the jar file
String libraryFileName = jniLibraryFileName;
is = getClass().getClassLoader().getResourceAsStream(libraryFileName);
if (is == null) {
// is there a fallback we can try
if (fallbackJniLibraryFileName == null) {
throw new RuntimeException(libraryFileName + " was not found inside JAR.");
}
// attempt to look up the fallback static library in the jar file
libraryFileName = fallbackJniLibraryFileName;
is = getClass().getClassLoader().getResourceAsStream(libraryFileName);
if (is == null) {
throw new RuntimeException(libraryFileName + " was not found inside JAR.");
}
}
temp = new File(parentDir, jniLibraryFileName);
if (temp.exists() && !temp.delete()) {
throw new RuntimeException("File: " + temp.getAbsolutePath()
+ " already exists and cannot be removed.");
// create a temporary file to copy the library to
final File temp;
if (tmpDir == null || tmpDir.isEmpty()) {
temp = File.createTempFile(tempFilePrefix, tempFileSuffix);
} else {
final File parentDir = new File(tmpDir);
if (!parentDir.exists()) {
throw new RuntimeException(
"Directory: " + parentDir.getAbsolutePath() + " does not exist!");
}
temp = new File(parentDir, libraryFileName);
if (temp.exists() && !temp.delete()) {
throw new RuntimeException(
"File: " + temp.getAbsolutePath() + " already exists and cannot be removed.");
}
if (!temp.createNewFile()) {
throw new RuntimeException("File: " + temp.getAbsolutePath() + " could not be created.");
}
}
if (!temp.createNewFile()) {
throw new RuntimeException("File: " + temp.getAbsolutePath()
+ " could not be created.");
if (!temp.exists()) {
throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist.");
} else {
temp.deleteOnExit();
}
}
if (!temp.exists()) {
throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist.");
} else {
temp.deleteOnExit();
}
// copy the library from the Jar file to the temp destination
Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
// attempt to copy the library from the Jar file to the temp destination
try (final InputStream is = getClass().getClassLoader().
getResourceAsStream(jniLibraryFileName)) {
if (is == null) {
throw new RuntimeException(jniLibraryFileName + " was not found inside JAR.");
} else {
Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
// return the temporary library file
return temp;
} finally {
if (is != null) {
is.close();
}
}
return temp;
}
/**

25
java/src/main/java/org/rocksdb/util/Environment.java
Unescape View File

@ -110,8 +110,14 @@ public class Environment {
return String.format("%sjni-linux%s%s", name, arch, getLibcPostfix());
}
} else if (isMac()) {
if (isAarch64()) {
return String.format("%sjni-osx-%s", name, ARCH);
if (is64Bit()) {
final String arch;
if (isAarch64()) {
arch = "arm64";
} else {
arch = "x86_64";
}
return String.format("%sjni-osx-%s", name, arch);
} else {
return String.format("%sjni-osx", name);
}
@ -131,10 +137,25 @@ public class Environment {
throw new UnsupportedOperationException(String.format("Cannot determine JNI library name for ARCH='%s' OS='%s' name='%s'", ARCH, OS, name));
}
public static /*@Nullable*/ String getFallbackJniLibraryName(final String name) {
if (isMac() && is64Bit()) {
return String.format("%sjni-osx", name);
}
return null;
}
public static String getJniLibraryFileName(final String name) {
return appendLibOsSuffix("lib" + getJniLibraryName(name), false);
}
public static /*@Nullable*/ String getFallbackJniLibraryFileName(final String name) {
final String fallbackJniLibraryName = getFallbackJniLibraryName(name);
if (fallbackJniLibraryName == null) {
return null;
}
return appendLibOsSuffix("lib" + fallbackJniLibraryName, false);
}
private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) {
if (isUnix() || isAix() || isSolaris() || isFreeBSD() || isOpenBSD()) {
return libraryFileName + ".so";

29
java/src/test/java/org/rocksdb/util/EnvironmentTest.java
Unescape View File

@ -9,7 +9,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import static org.assertj.core.api.Assertions.assertThat;
@ -37,18 +36,21 @@ public class EnvironmentTest {
isEqualTo(".jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-osx.jnilib");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.dylib");
}
@Test
public void mac64() {
setEnvironmentClassFields("mac", "x86-64");
public void mac64_x86_64() {
setEnvironmentClassFields("mac", "x86_64");
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-osx.jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx-x86_64.jnilib");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx.jnilib");
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.dylib");
}
@ -59,7 +61,9 @@ public class EnvironmentTest {
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).isEqualTo(".jnilib");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx-aarch64.jnilib");
.isEqualTo("librocksdbjni-osx-arm64.jnilib");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-osx.jnilib");
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.dylib");
}
@ -73,6 +77,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-linux32.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
// Linux musl-libc (Alpine)
@ -103,7 +108,8 @@ public class EnvironmentTest {
assertThat(Environment.isWindows()).isFalse();
assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".so");
Environment.getJniLibraryFileName("rocksdb");
assertThat(Environment.getJniLibraryFileName("rocksdb")).isEqualTo("blah");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
}
@Test
@ -115,6 +121,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-linux64.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
// Linux musl-libc (Alpine)
@ -124,6 +131,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-linux64-musl.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
// UNIX
@ -134,6 +142,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-linux64.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
// AIX
@ -143,6 +152,7 @@ public class EnvironmentTest {
isEqualTo(".so");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-aix64.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.so");
}
@ -161,6 +171,7 @@ public class EnvironmentTest {
isEqualTo(".dll");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-win64.dll");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.dll");
}
@ -177,6 +188,7 @@ public class EnvironmentTest {
assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-ppc64le");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-linux-ppc64le.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so");
// Linux musl-libc (Alpine)
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true);
@ -189,6 +201,7 @@ public class EnvironmentTest {
assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-ppc64le-musl");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-linux-ppc64le-musl.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so");
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false);
}
@ -205,6 +218,7 @@ public class EnvironmentTest {
assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-aarch64");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-linux-aarch64.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so");
// Linux musl-libc (Alpine)
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, true);
@ -217,6 +231,7 @@ public class EnvironmentTest {
assertThat(Environment.getJniLibraryName("rocksdb")).isEqualTo("rocksdbjni-linux-aarch64-musl");
assertThat(Environment.getJniLibraryFileName("rocksdb"))
.isEqualTo("librocksdbjni-linux-aarch64-musl.so");
assertThat(Environment.getFallbackJniLibraryFileName("rocksdb")).isNull();
assertThat(Environment.getSharedLibraryFileName("rocksdb")).isEqualTo("librocksdbjni.so");
setEnvironmentClassField(MUSL_LIBC_FIELD_NAME, false);
}

Write Preview
Loading…
Cancel
Save