Merge pull request #965 from koldat/jni_for_windows

Adding support for Windows JNI build
main
Siying Dong 9 years ago
commit c12ff20ab2
  1. 16
      CMakeLists.txt
  2. 155
      java/CMakeLists.txt
  3. 12
      java/rocksjni/backupablejni.cc
  4. 12
      java/rocksjni/backupenginejni.cc
  5. 5
      java/rocksjni/options.cc
  6. 5
      java/rocksjni/portal.h
  7. 13
      java/rocksjni/restorejni.cc
  8. 9
      java/rocksjni/rocksjni.cc
  9. 4
      java/rocksjni/statistics.cc
  10. 3
      java/rocksjni/write_batch_test.cc
  11. 16
      java/src/main/java/org/rocksdb/NativeLibraryLoader.java
  12. 7
      java/src/main/java/org/rocksdb/util/Environment.java
  13. 10
      java/src/test/java/org/rocksdb/BackupableDBOptionsTest.java
  14. 3
      java/src/test/java/org/rocksdb/InfoLogLevelTest.java
  15. 2
      java/src/test/java/org/rocksdb/NativeLibraryLoaderTest.java
  16. 18
      java/src/test/java/org/rocksdb/util/EnvironmentTest.java

@ -13,8 +13,8 @@
# cd build
# 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
# See thirdparty.inc for more information.
# sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 ..
# OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 ..
# sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
# OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
# 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
# or simply /m ot use all avail cores)
# msbuild rocksdb.sln
@ -243,6 +243,7 @@ set(SOURCES
util/xxhash.cc
utilities/backupable/backupable_db.cc
utilities/checkpoint/checkpoint.cc
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
utilities/document/document_db.cc
utilities/document/json_document.cc
utilities/document/json_document_builder.cc
@ -288,6 +289,17 @@ set_target_properties(rocksdb${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "-DROCK
add_dependencies(rocksdb${ARTIFACT_SUFFIX} GenerateBuildVersion)
target_link_libraries(rocksdb${ARTIFACT_SUFFIX} ${LIBS})
if (DEFINED JNI)
if (${JNI} EQUAL 1)
message(STATUS "JNI library is enabled")
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
else()
message(STATUS "JNI library is disabled")
endif()
else()
message(STATUS "JNI library is disabled")
endif()
set(APPS
db/db_bench.cc
db/memtablerep_bench.cc

@ -0,0 +1,155 @@
set(JNI_NATIVE_SOURCES
rocksjni/backupenginejni.cc
rocksjni/backupablejni.cc
rocksjni/checkpoint.cc
rocksjni/columnfamilyhandle.cc
rocksjni/compaction_filter.cc
rocksjni/comparator.cc
rocksjni/comparatorjnicallback.cc
rocksjni/env.cc
rocksjni/filter.cc
rocksjni/iterator.cc
rocksjni/loggerjnicallback.cc
rocksjni/memtablejni.cc
rocksjni/merge_operator.cc
rocksjni/options.cc
rocksjni/ratelimiterjni.cc
rocksjni/remove_emptyvalue_compactionfilterjni.cc
rocksjni/restorejni.cc
rocksjni/rocksjni.cc
rocksjni/slice.cc
rocksjni/snapshot.cc
rocksjni/statistics.cc
rocksjni/table.cc
rocksjni/transaction_log.cc
rocksjni/ttl.cc
rocksjni/write_batch.cc
rocksjni/writebatchhandlerjnicallback.cc
rocksjni/write_batch_with_index.cc
rocksjni/write_batch_test.cc
)
set(NATIVE_JAVA_CLASSES
org.rocksdb.AbstractCompactionFilter
org.rocksdb.AbstractComparator
org.rocksdb.AbstractSlice
org.rocksdb.BackupEngine
org.rocksdb.BackupableDB
org.rocksdb.BackupableDBOptions
org.rocksdb.BlockBasedTableConfig
org.rocksdb.BloomFilter
org.rocksdb.Checkpoint
org.rocksdb.ColumnFamilyHandle
org.rocksdb.ColumnFamilyOptions
org.rocksdb.Comparator
org.rocksdb.ComparatorOptions
org.rocksdb.DBOptions
org.rocksdb.DirectComparator
org.rocksdb.DirectSlice
org.rocksdb.Env
org.rocksdb.FlushOptions
org.rocksdb.Filter
org.rocksdb.GenericRateLimiterConfig
org.rocksdb.HashLinkedListMemTableConfig
org.rocksdb.HashSkipListMemTableConfig
org.rocksdb.Logger
org.rocksdb.MergeOperator
org.rocksdb.Options
org.rocksdb.PlainTableConfig
org.rocksdb.ReadOptions
org.rocksdb.RemoveEmptyValueCompactionFilter
org.rocksdb.RestoreBackupableDB
org.rocksdb.RestoreOptions
org.rocksdb.RocksDB
org.rocksdb.RocksEnv
org.rocksdb.RocksIterator
org.rocksdb.RocksMemEnv
org.rocksdb.SkipListMemTableConfig
org.rocksdb.Slice
org.rocksdb.Statistics
org.rocksdb.TransactionLogIterator
org.rocksdb.TtlDB
org.rocksdb.VectorMemTableConfig
org.rocksdb.Snapshot
org.rocksdb.StringAppendOperator
org.rocksdb.WriteBatch
org.rocksdb.WriteBatch.Handler
org.rocksdb.WriteOptions
org.rocksdb.WriteBatchWithIndex
org.rocksdb.WBWIRocksIterator
org.rocksdb.WriteBatchTest
org.rocksdb.WriteBatchTestInternalHelper
)
include_directories($ENV{JAVA_HOME}/include)
include_directories($ENV{JAVA_HOME}/include/win32)
include_directories(${PROJECT_SOURCE_DIR}/java)
set(JAVA_TEST_LIBDIR ${PROJECT_SOURCE_DIR}/java/test-libs)
set(JAVA_TMP_JAR ${JAVA_TEST_LIBDIR}/tmp.jar)
set(JAVA_JUNIT_JAR ${JAVA_TEST_LIBDIR}/junit-4.12.jar)
set(JAVA_HAMCR_JAR ${JAVA_TEST_LIBDIR}/hamcrest-core-1.3.jar)
set(JAVA_MOCKITO_JAR ${JAVA_TEST_LIBDIR}/mockito-all-1.10.19.jar)
set(JAVA_CGLIB_JAR ${JAVA_TEST_LIBDIR}/cglib-2.2.2.jar)
set(JAVA_ASSERTJ_JAR ${JAVA_TEST_LIBDIR}/assertj-core-1.7.1.jar)
set(JAVA_TESTCLASSPATH "${JAVA_JUNIT_JAR}\;${JAVA_HAMCR_JAR}\;${JAVA_MOCKITO_JAR}\;${JAVA_CGLIB_JAR}\;${JAVA_ASSERTJ_JAR}")
if(NOT EXISTS ${PROJECT_SOURCE_DIR}/java/classes)
execute_process(COMMAND mkdir ${PROJECT_SOURCE_DIR}/java/classes)
endif()
if(NOT EXISTS ${JAVA_TEST_LIBDIR})
execute_process(COMMAND mkdir ${JAVA_TEST_LIBDIR})
endif()
if(NOT EXISTS ${JAVA_JUNIT_JAR})
message("Downloading ${JAVA_JUNIT_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_JUNIT_JAR})
endif()
if(NOT EXISTS ${JAVA_HAMCR_JAR})
message("Downloading ${JAVA_HAMCR_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_HAMCR_JAR})
endif()
if(NOT EXISTS ${JAVA_MOCKITO_JAR})
message("Downloading ${JAVA_MOCKITO_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_MOCKITO_JAR})
endif()
if(NOT EXISTS ${JAVA_CGLIB_JAR})
message("Downloading ${JAVA_CGLIB_JAR}")
file(DOWNLOAD http://search.maven.org/remotecontent?filepath=cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_CGLIB_JAR})
endif()
if(NOT EXISTS ${JAVA_ASSERTJ_JAR})
message("Downloading ${JAVA_ASSERTJ_JAR}")
file(DOWNLOAD http://central.maven.org/maven2/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
list(GET downloadStatus 0 error_code)
if(NOT error_code EQUAL 0)
message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}")
endif()
file(RENAME ${JAVA_TMP_JAR} ${JAVA_ASSERTJ_JAR})
endif()
execute_process(COMMAND javac -cp ${JAVA_TESTCLASSPATH} -d ${PROJECT_SOURCE_DIR}/java/classes ${PROJECT_SOURCE_DIR}/java/src/main/java/org/rocksdb/util/*.java ${PROJECT_SOURCE_DIR}/java/src/main/java/org/rocksdb/*.java ${PROJECT_SOURCE_DIR}/java/src/test/java/org/rocksdb/*.java)
execute_process(COMMAND javah -cp ${PROJECT_SOURCE_DIR}/java/classes -d ${PROJECT_SOURCE_DIR}/java/include -jni ${NATIVE_JAVA_CLASSES})
add_library(rocksdbjni${ARTIFACT_SUFFIX} SHARED ${JNI_NATIVE_SOURCES})
set_target_properties(rocksdbjni${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/rocksdbjni${ARTIFACT_SUFFIX}.pdb")
target_link_libraries(rocksdbjni${ARTIFACT_SUFFIX} rocksdblib${ARTIFACT_SUFFIX} ${LIBS})

@ -103,20 +103,14 @@ jintArray Java_org_rocksdb_BackupableDB_getCorruptedBackups(
reinterpret_cast<rocksdb::BackupableDB*>(jhandle)->
GetCorruptedBackups(&backup_ids);
// store backupids in int array
const std::vector<rocksdb::BackupID>::size_type
kIdSize = backup_ids.size();
int int_backup_ids[kIdSize];
for (std::vector<rocksdb::BackupID>::size_type i = 0;
i != kIdSize; i++) {
int_backup_ids[i] = backup_ids[i];
}
std::vector<jint> int_backup_ids(backup_ids.begin(), backup_ids.end());
// Store ints in java array
jintArray ret_backup_ids;
// Its ok to loose precision here (64->32)
jsize ret_backup_ids_size = static_cast<jsize>(kIdSize);
jsize ret_backup_ids_size = static_cast<jsize>(backup_ids.size());
ret_backup_ids = env->NewIntArray(ret_backup_ids_size);
env->SetIntArrayRegion(ret_backup_ids, 0, ret_backup_ids_size,
int_backup_ids);
int_backup_ids.data());
return ret_backup_ids;
}

@ -81,20 +81,14 @@ jintArray Java_org_rocksdb_BackupEngine_getCorruptedBackups(
std::vector<rocksdb::BackupID> backup_ids;
backup_engine->GetCorruptedBackups(&backup_ids);
// store backupids in int array
const std::vector<rocksdb::BackupID>::size_type
kIdSize = backup_ids.size();
int int_backup_ids[kIdSize];
for (std::vector<rocksdb::BackupID>::size_type i = 0;
i != kIdSize; i++) {
int_backup_ids[i] = backup_ids[i];
}
std::vector<jint> int_backup_ids(backup_ids.begin(), backup_ids.end());
// Store ints in java array
jintArray ret_backup_ids;
// Its ok to loose precision here (64->32)
jsize ret_backup_ids_size = static_cast<jsize>(kIdSize);
jsize ret_backup_ids_size = static_cast<jsize>(backup_ids.size());
ret_backup_ids = env->NewIntArray(ret_backup_ids_size);
env->SetIntArrayRegion(ret_backup_ids, 0, ret_backup_ids_size,
int_backup_ids);
int_backup_ids.data());
return ret_backup_ids;
}

@ -8,7 +8,6 @@
#include <stdio.h>
#include <stdlib.h>
#include <jni.h>
#include <strings.h>
#include <memory>
#include "include/org_rocksdb_Options.h"
@ -1180,7 +1179,7 @@ jbyte Java_org_rocksdb_Options_compactionStyle(
void Java_org_rocksdb_Options_setMaxTableFilesSizeFIFO(
JNIEnv* env, jobject jobj, jlong jhandle, jlong jmax_table_files_size) {
reinterpret_cast<rocksdb::Options*>(jhandle)->compaction_options_fifo.max_table_files_size =
static_cast<long>(jmax_table_files_size);
static_cast<uint64_t>(jmax_table_files_size);
}
/*
@ -2339,7 +2338,7 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_compactionStyle(
void Java_org_rocksdb_ColumnFamilyOptions_setMaxTableFilesSizeFIFO(
JNIEnv* env, jobject jobj, jlong jhandle, jlong jmax_table_files_size) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)->compaction_options_fifo.max_table_files_size =
static_cast<long>(jmax_table_files_size);
static_cast<uint64_t>(jmax_table_files_size);
}
/*

@ -24,6 +24,11 @@
#include "rocksjni/loggerjnicallback.h"
#include "rocksjni/writebatchhandlerjnicallback.h"
// Remove macro on windows
#ifdef DELETE
#undef DELETE
#endif
namespace rocksdb {
// Detect if jlong overflows size_t

@ -156,21 +156,14 @@ jintArray Java_org_rocksdb_RestoreBackupableDB_getCorruptedBackups(
reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle)->
GetCorruptedBackups(&backup_ids);
// store backupids in int array
const std::vector<rocksdb::BackupID>::size_type
kIdSize = backup_ids.size();
int int_backup_ids[kIdSize];
for (std::vector<rocksdb::BackupID>::size_type i = 0;
i != kIdSize; i++) {
int_backup_ids[i] = backup_ids[i];
}
std::vector<jint> int_backup_ids(backup_ids.begin(), backup_ids.end());
// Store ints in java array
jintArray ret_backup_ids;
// Its ok to loose precision here (64->32)
jsize ret_backup_ids_size = static_cast<jsize>(kIdSize);
jsize ret_backup_ids_size = static_cast<jsize>(backup_ids.size());
ret_backup_ids = env->NewIntArray(ret_backup_ids_size);
env->SetIntArrayRegion(ret_backup_ids, 0, ret_backup_ids_size,
int_backup_ids);
int_backup_ids.data());
return ret_backup_ids;
}

@ -12,6 +12,7 @@
#include <memory>
#include <string>
#include <vector>
#include <algorithm>
#include "include/org_rocksdb_RocksDB.h"
#include "rocksdb/db.h"
@ -19,6 +20,10 @@
#include "rocksdb/types.h"
#include "rocksjni/portal.h"
#ifdef min
#undef min
#endif
//////////////////////////////////////////////////////////////////////////////
// rocksdb::DB::Open
@ -688,8 +693,8 @@ jint rocksdb_get_helper(
return kStatusError;
}
int cvalue_len = static_cast<int>(cvalue.size());
int length = std::min(jentry_value_len, cvalue_len);
jint cvalue_len = static_cast<jint>(cvalue.size());
jint length = std::min(jentry_value_len, cvalue_len);
env->SetByteArrayRegion(
jentry_value, 0, length,

@ -20,7 +20,7 @@
* Signature: (IJ)J
*/
jlong Java_org_rocksdb_Statistics_getTickerCount0(
JNIEnv* env, jobject jobj, int tickerType, jlong handle) {
JNIEnv* env, jobject jobj, jint tickerType, jlong handle) {
auto st = reinterpret_cast<rocksdb::Statistics*>(handle);
assert(st != nullptr);
@ -33,7 +33,7 @@ jlong Java_org_rocksdb_Statistics_getTickerCount0(
* Signature: (IJ)Lorg/rocksdb/HistogramData;
*/
jobject Java_org_rocksdb_Statistics_geHistogramData0(
JNIEnv* env, jobject jobj, int histogramType, jlong handle) {
JNIEnv* env, jobject jobj, jint histogramType, jlong handle) {
auto st = reinterpret_cast<rocksdb::Statistics*>(handle);
assert(st != nullptr);

@ -60,7 +60,8 @@ jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
rocksdb::ParsedInternalKey ikey;
memset(reinterpret_cast<void*>(&ikey), 0, sizeof(ikey));
assert(rocksdb::ParseInternalKey(iter->key(), &ikey));
bool parsed = rocksdb::ParseInternalKey(iter->key(), &ikey);
assert(parsed);
switch (ikey.type) {
case rocksdb::kTypeValue:
state.append("Put(");

@ -19,7 +19,7 @@ public class NativeLibraryLoader {
private static final String jniLibraryName = Environment.getJniLibraryName("rocksdb");
private static final String jniLibraryFileName = Environment.getJniLibraryFileName("rocksdb");
private static final String tempFilePrefix = "librocksdbjni";
private static final String tempFileSuffix = "." + Environment.getJniLibraryExtension();
private static final String tempFileSuffix = Environment.getJniLibraryExtension();
/**
* Get a reference to the NativeLibraryLoader
@ -75,8 +75,15 @@ public class NativeLibraryLoader {
void loadLibraryFromJar(final String tmpDir)
throws IOException {
if (!initialized) {
System.load(loadLibraryFromJarToTemp(tmpDir).getAbsolutePath());
initialized = true;
}
}
File loadLibraryFromJarToTemp(final String tmpDir)
throws IOException {
final File temp;
if (tmpDir == null || tmpDir.equals("")) {
if (tmpDir == null || tmpDir.isEmpty()) {
temp = File.createTempFile(tempFilePrefix, tempFileSuffix);
} else {
temp = new File(tmpDir, jniLibraryFileName);
@ -102,10 +109,9 @@ public class NativeLibraryLoader {
}
}
System.load(temp.getAbsolutePath());
initialized = true;
}
return temp;
}
/**
* Private constructor to disallow instantiation
*/

@ -42,6 +42,8 @@ public class Environment {
return String.format("%sjni-osx", name);
} else if (isSolaris()) {
return String.format("%sjni-solaris%d", name, is64Bit() ? 64 : 32);
} else if (isWindows() && is64Bit()) {
return String.format("%sjni-win64", name);
}
throw new UnsupportedOperationException();
}
@ -55,11 +57,16 @@ public class Environment {
return libraryFileName + ".so";
} else if (isMac()) {
return libraryFileName + (shared ? ".dylib" : ".jnilib");
} else if (isWindows()) {
return libraryFileName + ".dll";
}
throw new UnsupportedOperationException();
}
public static String getJniLibraryExtension() {
if (isWindows()) {
return ".dll";
}
return (isMac()) ? ".jnilib" : ".so";
}
}

@ -5,18 +5,18 @@
package org.rocksdb;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Random;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
public class BackupableDBOptionsTest {
private final static String ARBITRARY_PATH = "/tmp";
private final static String ARBITRARY_PATH = System.getProperty("java.io.tmpdir");
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =

@ -4,6 +4,7 @@ import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.rocksdb.util.Environment;
import java.io.IOException;
@ -113,7 +114,7 @@ public class InfoLogLevelTest {
* @throws IOException if file is not found.
*/
private String getLogContentsWithoutHeader() throws IOException {
final String separator = System.getProperty("line.separator");
final String separator = Environment.isWindows() ? "\n" : System.getProperty("line.separator");
final String[] lines = new String(readAllBytes(get(
dbFolder.getRoot().getAbsolutePath()+ "/LOG"))).split(separator);

@ -21,7 +21,7 @@ public class NativeLibraryLoaderTest {
@Test
public void tempFolder() throws IOException {
NativeLibraryLoader.getInstance().loadLibraryFromJar(
NativeLibraryLoader.getInstance().loadLibraryFromJarToTemp(
temporaryFolder.getRoot().getAbsolutePath());
Path path = Paths.get(temporaryFolder.getRoot().getAbsolutePath(),
Environment.getJniLibraryFileName("rocksdb"));

@ -117,16 +117,22 @@ public class EnvironmentTest {
assertThat(Environment.isWindows()).isTrue();
}
@Test(expected = UnsupportedOperationException.class)
public void failWinJniLibraryName(){
@Test
public void win64() {
setEnvironmentClassFields("win", "x64");
Environment.getJniLibraryFileName("rocksdb");
assertThat(Environment.isWindows()).isTrue();
assertThat(Environment.getJniLibraryExtension()).
isEqualTo(".dll");
assertThat(Environment.getJniLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni-win64.dll");
assertThat(Environment.getSharedLibraryFileName("rocksdb")).
isEqualTo("librocksdbjni.dll");
}
@Test(expected = UnsupportedOperationException.class)
public void failWinSharedLibrary(){
setEnvironmentClassFields("win", "x64");
Environment.getSharedLibraryFileName("rocksdb");
public void win32(){
setEnvironmentClassFields("win", "32");
Environment.getJniLibraryFileName("rocksdb");
}
private void setEnvironmentClassFields(String osName,

Loading…
Cancel
Save