Summary: JNI wrappers for LoadLatestOptions, LoadOptionsFromFile and GetLatestOptionsFileName APIs. Closes https://github.com/facebook/rocksdb/pull/2898 Differential Revision: D5857934 Pulled By: sagar0 fbshipit-source-id: 68b79e83eab8de9416e3f1fef73e11cf7947e90amain
Sagar Vemuri
7 years ago
committed by
Facebook Github Bot
parent
96a13b4f4b
commit
c8f3606731
@ -0,0 +1,114 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
// This file implements the "bridge" between Java and C++ and enables
|
||||
// calling C++ rocksdb::OptionsUtil methods from Java side.
|
||||
|
||||
#include <jni.h> |
||||
|
||||
#include "include/org_rocksdb_OptionsUtil.h" |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/utilities/options_util.h" |
||||
#include "rocksjni/portal.h" |
||||
|
||||
void build_column_family_descriptor_list( |
||||
JNIEnv* env, jobject jcfds, |
||||
std::vector<rocksdb::ColumnFamilyDescriptor>& cf_descs) { |
||||
jmethodID add_mid = rocksdb::ListJni::getListAddMethodId(env); |
||||
if (add_mid == nullptr) { |
||||
// exception occurred accessing method
|
||||
return; |
||||
} |
||||
|
||||
// Column family descriptor
|
||||
for (rocksdb::ColumnFamilyDescriptor& cfd : cf_descs) { |
||||
// Construct a ColumnFamilyDescriptor java object
|
||||
jobject jcfd = rocksdb::ColumnFamilyDescriptorJni::construct(env, &cfd); |
||||
if (env->ExceptionCheck()) { |
||||
// exception occurred constructing object
|
||||
if (jcfd != nullptr) { |
||||
env->DeleteLocalRef(jcfd); |
||||
} |
||||
return; |
||||
} |
||||
|
||||
// Add the object to java list.
|
||||
jboolean rs = env->CallBooleanMethod(jcfds, add_mid, jcfd); |
||||
if (env->ExceptionCheck() || rs == JNI_FALSE) { |
||||
// exception occurred calling method, or could not add
|
||||
if (jcfd != nullptr) { |
||||
env->DeleteLocalRef(jcfd); |
||||
} |
||||
return; |
||||
} |
||||
} |
||||
} |
||||
|
||||
/*
|
||||
* Class: org_rocksdb_OptionsUtil |
||||
* Method: loadLatestOptions |
||||
* Signature: (Ljava/lang/String;JLjava/util/List;Z)V |
||||
*/ |
||||
void Java_org_rocksdb_OptionsUtil_loadLatestOptions( |
||||
JNIEnv* env, jclass jcls, jstring jdbpath, jlong jenv_handle, |
||||
jlong jdb_opts_handle, jobject jcfds, jboolean ignore_unknown_options) { |
||||
const char* db_path = env->GetStringUTFChars(jdbpath, nullptr); |
||||
std::vector<rocksdb::ColumnFamilyDescriptor> cf_descs; |
||||
rocksdb::Status s = rocksdb::LoadLatestOptions( |
||||
db_path, reinterpret_cast<rocksdb::Env*>(jenv_handle), |
||||
reinterpret_cast<rocksdb::DBOptions*>(jdb_opts_handle), &cf_descs, |
||||
ignore_unknown_options); |
||||
env->ReleaseStringUTFChars(jdbpath, db_path); |
||||
|
||||
if (!s.ok()) { |
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s); |
||||
} |
||||
|
||||
build_column_family_descriptor_list(env, jcfds, cf_descs); |
||||
} |
||||
|
||||
/*
|
||||
* Class: org_rocksdb_OptionsUtil |
||||
* Method: loadOptionsFromFile |
||||
* Signature: (Ljava/lang/String;JJLjava/util/List;Z)V |
||||
*/ |
||||
void Java_org_rocksdb_OptionsUtil_loadOptionsFromFile( |
||||
JNIEnv* env, jclass jcls, jstring jopts_file_name, jlong jenv_handle, |
||||
jlong jdb_opts_handle, jobject jcfds, jboolean ignore_unknown_options) { |
||||
const char* opts_file_name = env->GetStringUTFChars(jopts_file_name, nullptr); |
||||
std::vector<rocksdb::ColumnFamilyDescriptor> cf_descs; |
||||
rocksdb::Status s = rocksdb::LoadOptionsFromFile( |
||||
opts_file_name, reinterpret_cast<rocksdb::Env*>(jenv_handle), |
||||
reinterpret_cast<rocksdb::DBOptions*>(jdb_opts_handle), &cf_descs, |
||||
ignore_unknown_options); |
||||
env->ReleaseStringUTFChars(jopts_file_name, opts_file_name); |
||||
|
||||
if (!s.ok()) { |
||||
rocksdb::RocksDBExceptionJni::ThrowNew(env, s); |
||||
} |
||||
|
||||
build_column_family_descriptor_list(env, jcfds, cf_descs); |
||||
} |
||||
|
||||
/*
|
||||
* Class: org_rocksdb_OptionsUtil |
||||
* Method: getLatestOptionsFileName |
||||
* Signature: (Ljava/lang/String;J)Ljava/lang/String; |
||||
*/ |
||||
jstring Java_org_rocksdb_OptionsUtil_getLatestOptionsFileName( |
||||
JNIEnv* env, jclass jcls, jstring jdbpath, jlong jenv_handle) { |
||||
const char* db_path = env->GetStringUTFChars(jdbpath, nullptr); |
||||
std::string options_file_name; |
||||
if (db_path != nullptr) { |
||||
rocksdb::GetLatestOptionsFileName( |
||||
db_path, reinterpret_cast<rocksdb::Env*>(jenv_handle), |
||||
&options_file_name); |
||||
} |
||||
env->ReleaseStringUTFChars(jdbpath, db_path); |
||||
|
||||
return env->NewStringUTF(options_file_name.c_str()); |
||||
} |
||||
@ -0,0 +1,142 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
package org.rocksdb; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
public class OptionsUtil { |
||||
/** |
||||
* A static method to construct the DBOptions and ColumnFamilyDescriptors by |
||||
* loading the latest RocksDB options file stored in the specified rocksdb |
||||
* database. |
||||
* |
||||
* Note that the all the pointer options (except table_factory, which will |
||||
* be described in more details below) will be initialized with the default |
||||
* values. Developers can further initialize them after this function call. |
||||
* Below is an example list of pointer options which will be initialized. |
||||
* |
||||
* - env |
||||
* - memtable_factory |
||||
* - compaction_filter_factory |
||||
* - prefix_extractor |
||||
* - comparator |
||||
* - merge_operator |
||||
* - compaction_filter |
||||
* |
||||
* For table_factory, this function further supports deserializing |
||||
* BlockBasedTableFactory and its BlockBasedTableOptions except the |
||||
* pointer options of BlockBasedTableOptions (flush_block_policy_factory, |
||||
* block_cache, and block_cache_compressed), which will be initialized with |
||||
* default values. Developers can further specify these three options by |
||||
* casting the return value of TableFactoroy::GetOptions() to |
||||
* BlockBasedTableOptions and making necessary changes. |
||||
* |
||||
* @param dbPath the path to the RocksDB. |
||||
* @param env {@link org.rocksdb.Env} instance. |
||||
* @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be |
||||
* filled and returned. |
||||
* @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be |
||||
* returned. |
||||
* |
||||
* @throws RocksDBException thrown if error happens in underlying |
||||
* native library. |
||||
*/ |
||||
|
||||
public static void loadLatestOptions(String dbPath, Env env, DBOptions dbOptions, |
||||
List<ColumnFamilyDescriptor> cfDescs) throws RocksDBException { |
||||
loadLatestOptions(dbPath, env, dbOptions, cfDescs, false); |
||||
} |
||||
|
||||
/** |
||||
* @param dbPath the path to the RocksDB. |
||||
* @param env {@link org.rocksdb.Env} instance. |
||||
* @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be |
||||
* filled and returned. |
||||
* @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be |
||||
* returned. |
||||
* @param ignoreUnknownOptions this flag can be set to true if you want to |
||||
* ignore options that are from a newer version of the db, esentially for |
||||
* forward compatibility. |
||||
* |
||||
* @throws RocksDBException thrown if error happens in underlying |
||||
* native library. |
||||
*/ |
||||
public static void loadLatestOptions(String dbPath, Env env, DBOptions dbOptions, |
||||
List<ColumnFamilyDescriptor> cfDescs, boolean ignoreUnknownOptions) throws RocksDBException { |
||||
loadLatestOptions( |
||||
dbPath, env.nativeHandle_, dbOptions.nativeHandle_, cfDescs, ignoreUnknownOptions); |
||||
} |
||||
|
||||
/** |
||||
* Similar to LoadLatestOptions, this function constructs the DBOptions |
||||
* and ColumnFamilyDescriptors based on the specified RocksDB Options file. |
||||
* See LoadLatestOptions above. |
||||
* |
||||
* @param optionsFileName the RocksDB options file path. |
||||
* @param env {@link org.rocksdb.Env} instance. |
||||
* @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be |
||||
* filled and returned. |
||||
* @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be |
||||
* returned. |
||||
* |
||||
* @throws RocksDBException thrown if error happens in underlying |
||||
* native library. |
||||
*/ |
||||
public static void loadOptionsFromFile(String optionsFileName, Env env, DBOptions dbOptions, |
||||
List<ColumnFamilyDescriptor> cfDescs) throws RocksDBException { |
||||
loadOptionsFromFile(optionsFileName, env, dbOptions, cfDescs, false); |
||||
} |
||||
|
||||
/** |
||||
* @param optionsFileName the RocksDB options file path. |
||||
* @param env {@link org.rocksdb.Env} instance. |
||||
* @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be |
||||
* filled and returned. |
||||
* @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be |
||||
* returned. |
||||
* @param ignoreUnknownOptions this flag can be set to true if you want to |
||||
* ignore options that are from a newer version of the db, esentially for |
||||
* forward compatibility. |
||||
* |
||||
* @throws RocksDBException thrown if error happens in underlying |
||||
* native library. |
||||
*/ |
||||
public static void loadOptionsFromFile(String optionsFileName, Env env, DBOptions dbOptions, |
||||
List<ColumnFamilyDescriptor> cfDescs, boolean ignoreUnknownOptions) throws RocksDBException { |
||||
loadOptionsFromFile( |
||||
optionsFileName, env.nativeHandle_, dbOptions.nativeHandle_, cfDescs, ignoreUnknownOptions); |
||||
} |
||||
|
||||
/** |
||||
* Returns the latest options file name under the specified RocksDB path. |
||||
* |
||||
* @param dbPath the path to the RocksDB. |
||||
* @param env {@link org.rocksdb.Env} instance. |
||||
* @return the latest options file name under the db path. |
||||
* |
||||
* @throws RocksDBException thrown if error happens in underlying |
||||
* native library. |
||||
*/ |
||||
public static String getLatestOptionsFileName(String dbPath, Env env) throws RocksDBException { |
||||
return getLatestOptionsFileName(dbPath, env.nativeHandle_); |
||||
} |
||||
|
||||
/** |
||||
* Private constructor. |
||||
* This class has only static methods and shouldn't be instantiated. |
||||
*/ |
||||
private OptionsUtil() {} |
||||
|
||||
// native methods
|
||||
private native static void loadLatestOptions(String dbPath, long envHandle, long dbOptionsHandle, |
||||
List<ColumnFamilyDescriptor> cfDescs, boolean ignoreUnknownOptions) throws RocksDBException; |
||||
private native static void loadOptionsFromFile(String optionsFileName, long envHandle, |
||||
long dbOptionsHandle, List<ColumnFamilyDescriptor> cfDescs, boolean ignoreUnknownOptions) |
||||
throws RocksDBException; |
||||
private native static String getLatestOptionsFileName(String dbPath, long envHandle) |
||||
throws RocksDBException; |
||||
} |
||||
@ -0,0 +1,126 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
package org.rocksdb; |
||||
|
||||
import org.junit.ClassRule; |
||||
import org.junit.Rule; |
||||
import org.junit.Test; |
||||
import org.junit.rules.TemporaryFolder; |
||||
|
||||
import java.util.*; |
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat; |
||||
|
||||
public class OptionsUtilTest { |
||||
@ClassRule |
||||
public static final RocksMemoryResource rocksMemoryResource = new RocksMemoryResource(); |
||||
|
||||
@Rule public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||
|
||||
enum TestAPI { LOAD_LATEST_OPTIONS, LOAD_OPTIONS_FROM_FILE } |
||||
|
||||
@Test |
||||
public void loadLatestOptions() throws RocksDBException { |
||||
verifyOptions(TestAPI.LOAD_LATEST_OPTIONS); |
||||
} |
||||
|
||||
@Test |
||||
public void loadOptionsFromFile() throws RocksDBException { |
||||
verifyOptions(TestAPI.LOAD_OPTIONS_FROM_FILE); |
||||
} |
||||
|
||||
@Test |
||||
public void getLatestOptionsFileName() throws RocksDBException { |
||||
final String dbPath = dbFolder.getRoot().getAbsolutePath(); |
||||
try (final Options options = new Options().setCreateIfMissing(true); |
||||
final RocksDB db = RocksDB.open(options, dbPath)) { |
||||
assertThat(db).isNotNull(); |
||||
} |
||||
|
||||
String fName = OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); |
||||
assertThat(fName).isNotNull(); |
||||
assert(fName.startsWith("OPTIONS-") == true); |
||||
// System.out.println("latest options fileName: " + fName);
|
||||
} |
||||
|
||||
private void verifyOptions(TestAPI apiType) throws RocksDBException { |
||||
final String dbPath = dbFolder.getRoot().getAbsolutePath(); |
||||
final Options options = new Options() |
||||
.setCreateIfMissing(true) |
||||
.setParanoidChecks(false) |
||||
.setMaxOpenFiles(478) |
||||
.setDelayedWriteRate(1234567L); |
||||
final ColumnFamilyOptions baseDefaultCFOpts = new ColumnFamilyOptions(); |
||||
final byte[] secondCFName = "new_cf".getBytes(); |
||||
final ColumnFamilyOptions baseSecondCFOpts = |
||||
new ColumnFamilyOptions() |
||||
.setWriteBufferSize(70 * 1024) |
||||
.setMaxWriteBufferNumber(7) |
||||
.setMaxBytesForLevelBase(53 * 1024 * 1024) |
||||
.setLevel0FileNumCompactionTrigger(3) |
||||
.setLevel0SlowdownWritesTrigger(51) |
||||
.setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); |
||||
|
||||
// Create a database with a new column family
|
||||
try (final RocksDB db = RocksDB.open(options, dbPath)) { |
||||
assertThat(db).isNotNull(); |
||||
|
||||
// create column family
|
||||
try (final ColumnFamilyHandle columnFamilyHandle = |
||||
db.createColumnFamily(new ColumnFamilyDescriptor(secondCFName, baseSecondCFOpts))) { |
||||
assert(columnFamilyHandle != null); |
||||
} |
||||
} |
||||
|
||||
// Read the options back and verify
|
||||
DBOptions dbOptions = new DBOptions(); |
||||
final List<ColumnFamilyDescriptor> cfDescs = new ArrayList<>(); |
||||
String path = dbPath; |
||||
if (apiType == TestAPI.LOAD_LATEST_OPTIONS) { |
||||
OptionsUtil.loadLatestOptions(path, Env.getDefault(), dbOptions, cfDescs, false); |
||||
} else if (apiType == TestAPI.LOAD_OPTIONS_FROM_FILE) { |
||||
path = dbPath + "/" + OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); |
||||
OptionsUtil.loadOptionsFromFile(path, Env.getDefault(), dbOptions, cfDescs, false); |
||||
} |
||||
|
||||
assertThat(dbOptions.createIfMissing()).isEqualTo(options.createIfMissing()); |
||||
assertThat(dbOptions.paranoidChecks()).isEqualTo(options.paranoidChecks()); |
||||
assertThat(dbOptions.maxOpenFiles()).isEqualTo(options.maxOpenFiles()); |
||||
assertThat(dbOptions.delayedWriteRate()).isEqualTo(options.delayedWriteRate()); |
||||
|
||||
assertThat(cfDescs.size()).isEqualTo(2); |
||||
assertThat(cfDescs.get(0)).isNotNull(); |
||||
assertThat(cfDescs.get(1)).isNotNull(); |
||||
assertThat(cfDescs.get(0).columnFamilyName()).isEqualTo(RocksDB.DEFAULT_COLUMN_FAMILY); |
||||
assertThat(cfDescs.get(1).columnFamilyName()).isEqualTo(secondCFName); |
||||
|
||||
ColumnFamilyOptions defaultCFOpts = cfDescs.get(0).columnFamilyOptions(); |
||||
assertThat(defaultCFOpts.writeBufferSize()).isEqualTo(baseDefaultCFOpts.writeBufferSize()); |
||||
assertThat(defaultCFOpts.maxWriteBufferNumber()) |
||||
.isEqualTo(baseDefaultCFOpts.maxWriteBufferNumber()); |
||||
assertThat(defaultCFOpts.maxBytesForLevelBase()) |
||||
.isEqualTo(baseDefaultCFOpts.maxBytesForLevelBase()); |
||||
assertThat(defaultCFOpts.level0FileNumCompactionTrigger()) |
||||
.isEqualTo(baseDefaultCFOpts.level0FileNumCompactionTrigger()); |
||||
assertThat(defaultCFOpts.level0SlowdownWritesTrigger()) |
||||
.isEqualTo(baseDefaultCFOpts.level0SlowdownWritesTrigger()); |
||||
assertThat(defaultCFOpts.bottommostCompressionType()) |
||||
.isEqualTo(baseDefaultCFOpts.bottommostCompressionType()); |
||||
|
||||
ColumnFamilyOptions secondCFOpts = cfDescs.get(1).columnFamilyOptions(); |
||||
assertThat(secondCFOpts.writeBufferSize()).isEqualTo(baseSecondCFOpts.writeBufferSize()); |
||||
assertThat(secondCFOpts.maxWriteBufferNumber()) |
||||
.isEqualTo(baseSecondCFOpts.maxWriteBufferNumber()); |
||||
assertThat(secondCFOpts.maxBytesForLevelBase()) |
||||
.isEqualTo(baseSecondCFOpts.maxBytesForLevelBase()); |
||||
assertThat(secondCFOpts.level0FileNumCompactionTrigger()) |
||||
.isEqualTo(baseSecondCFOpts.level0FileNumCompactionTrigger()); |
||||
assertThat(secondCFOpts.level0SlowdownWritesTrigger()) |
||||
.isEqualTo(baseSecondCFOpts.level0SlowdownWritesTrigger()); |
||||
assertThat(secondCFOpts.bottommostCompressionType()) |
||||
.isEqualTo(baseSecondCFOpts.bottommostCompressionType()); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue