Summary: JNI wrappers for LoadLatestOptions, LoadOptionsFromFile and GetLatestOptionsFileName APIs. Closes https://github.com/facebook/rocksdb/pull/2898 Differential Revision: D5857934 Pulled By: sagar0 fbshipit-source-id: 68b79e83eab8de9416e3f1fef73e11cf7947e90amain
Sagar Vemuri
7 years ago
committed by
Facebook Github Bot
parent
96a13b4f4b
commit
c8f3606731
@ -0,0 +1,114 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
//
|
||||||
|
// This file implements the "bridge" between Java and C++ and enables
|
||||||
|
// calling C++ rocksdb::OptionsUtil methods from Java side.
|
||||||
|
|
||||||
|
#include <jni.h> |
||||||
|
|
||||||
|
#include "include/org_rocksdb_OptionsUtil.h" |
||||||
|
|
||||||
|
#include "rocksdb/db.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/utilities/options_util.h" |
||||||
|
#include "rocksjni/portal.h" |
||||||
|
|
||||||
|
void build_column_family_descriptor_list( |
||||||
|
JNIEnv* env, jobject jcfds, |
||||||
|
std::vector<rocksdb::ColumnFamilyDescriptor>& cf_descs) { |
||||||
|
jmethodID add_mid = rocksdb::ListJni::getListAddMethodId(env); |
||||||
|
if (add_mid == nullptr) { |
||||||
|
// exception occurred accessing method
|
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
// Column family descriptor
|
||||||
|
for (rocksdb::ColumnFamilyDescriptor& cfd : cf_descs) { |
||||||
|
// Construct a ColumnFamilyDescriptor java object
|
||||||
|
jobject jcfd = rocksdb::ColumnFamilyDescriptorJni::construct(env, &cfd); |
||||||
|
if (env->ExceptionCheck()) { |
||||||
|
// exception occurred constructing object
|
||||||
|
if (jcfd != nullptr) { |
||||||
|
env->DeleteLocalRef(jcfd); |
||||||
|
} |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
// Add the object to java list.
|
||||||
|
jboolean rs = env->CallBooleanMethod(jcfds, add_mid, jcfd); |
||||||
|
if (env->ExceptionCheck() || rs == JNI_FALSE) { |
||||||
|
// exception occurred calling method, or could not add
|
||||||
|
if (jcfd != nullptr) { |
||||||
|
env->DeleteLocalRef(jcfd); |
||||||
|
} |
||||||
|
return; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_OptionsUtil |
||||||
|
* Method: loadLatestOptions |
||||||
|
* Signature: (Ljava/lang/String;JLjava/util/List;Z)V |
||||||
|
*/ |
||||||
|
void Java_org_rocksdb_OptionsUtil_loadLatestOptions( |
||||||
|
JNIEnv* env, jclass jcls, jstring jdbpath, jlong jenv_handle, |
||||||
|
jlong jdb_opts_handle, jobject jcfds, jboolean ignore_unknown_options) { |
||||||
|
const char* db_path = env->GetStringUTFChars(jdbpath, nullptr); |
||||||
|
std::vector<rocksdb::ColumnFamilyDescriptor> cf_descs; |
||||||
|
rocksdb::Status s = rocksdb::LoadLatestOptions( |
||||||
|
db_path, reinterpret_cast<rocksdb::Env*>(jenv_handle), |
||||||
|
reinterpret_cast<rocksdb::DBOptions*>(jdb_opts_handle), &cf_descs, |
||||||
|
ignore_unknown_options); |
||||||
|
env->ReleaseStringUTFChars(jdbpath, db_path); |
||||||
|
|
||||||
|
if (!s.ok()) { |
||||||
|
rocksdb::RocksDBExceptionJni::ThrowNew(env, s); |
||||||
|
} |
||||||
|
|
||||||
|
build_column_family_descriptor_list(env, jcfds, cf_descs); |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_OptionsUtil |
||||||
|
* Method: loadOptionsFromFile |
||||||
|
* Signature: (Ljava/lang/String;JJLjava/util/List;Z)V |
||||||
|
*/ |
||||||
|
void Java_org_rocksdb_OptionsUtil_loadOptionsFromFile( |
||||||
|
JNIEnv* env, jclass jcls, jstring jopts_file_name, jlong jenv_handle, |
||||||
|
jlong jdb_opts_handle, jobject jcfds, jboolean ignore_unknown_options) { |
||||||
|
const char* opts_file_name = env->GetStringUTFChars(jopts_file_name, nullptr); |
||||||
|
std::vector<rocksdb::ColumnFamilyDescriptor> cf_descs; |
||||||
|
rocksdb::Status s = rocksdb::LoadOptionsFromFile( |
||||||
|
opts_file_name, reinterpret_cast<rocksdb::Env*>(jenv_handle), |
||||||
|
reinterpret_cast<rocksdb::DBOptions*>(jdb_opts_handle), &cf_descs, |
||||||
|
ignore_unknown_options); |
||||||
|
env->ReleaseStringUTFChars(jopts_file_name, opts_file_name); |
||||||
|
|
||||||
|
if (!s.ok()) { |
||||||
|
rocksdb::RocksDBExceptionJni::ThrowNew(env, s); |
||||||
|
} |
||||||
|
|
||||||
|
build_column_family_descriptor_list(env, jcfds, cf_descs); |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_OptionsUtil |
||||||
|
* Method: getLatestOptionsFileName |
||||||
|
* Signature: (Ljava/lang/String;J)Ljava/lang/String; |
||||||
|
*/ |
||||||
|
jstring Java_org_rocksdb_OptionsUtil_getLatestOptionsFileName( |
||||||
|
JNIEnv* env, jclass jcls, jstring jdbpath, jlong jenv_handle) { |
||||||
|
const char* db_path = env->GetStringUTFChars(jdbpath, nullptr); |
||||||
|
std::string options_file_name; |
||||||
|
if (db_path != nullptr) { |
||||||
|
rocksdb::GetLatestOptionsFileName( |
||||||
|
db_path, reinterpret_cast<rocksdb::Env*>(jenv_handle), |
||||||
|
&options_file_name); |
||||||
|
} |
||||||
|
env->ReleaseStringUTFChars(jdbpath, db_path); |
||||||
|
|
||||||
|
return env->NewStringUTF(options_file_name.c_str()); |
||||||
|
} |
||||||
@ -0,0 +1,142 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.List; |
||||||
|
|
||||||
|
public class OptionsUtil { |
||||||
|
/** |
||||||
|
* A static method to construct the DBOptions and ColumnFamilyDescriptors by |
||||||
|
* loading the latest RocksDB options file stored in the specified rocksdb |
||||||
|
* database. |
||||||
|
* |
||||||
|
* Note that the all the pointer options (except table_factory, which will |
||||||
|
* be described in more details below) will be initialized with the default |
||||||
|
* values. Developers can further initialize them after this function call. |
||||||
|
* Below is an example list of pointer options which will be initialized. |
||||||
|
* |
||||||
|
* - env |
||||||
|
* - memtable_factory |
||||||
|
* - compaction_filter_factory |
||||||
|
* - prefix_extractor |
||||||
|
* - comparator |
||||||
|
* - merge_operator |
||||||
|
* - compaction_filter |
||||||
|
* |
||||||
|
* For table_factory, this function further supports deserializing |
||||||
|
* BlockBasedTableFactory and its BlockBasedTableOptions except the |
||||||
|
* pointer options of BlockBasedTableOptions (flush_block_policy_factory, |
||||||
|
* block_cache, and block_cache_compressed), which will be initialized with |
||||||
|
* default values. Developers can further specify these three options by |
||||||
|
* casting the return value of TableFactoroy::GetOptions() to |
||||||
|
* BlockBasedTableOptions and making necessary changes. |
||||||
|
* |
||||||
|
* @param dbPath the path to the RocksDB. |
||||||
|
* @param env {@link org.rocksdb.Env} instance. |
||||||
|
* @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be |
||||||
|
* filled and returned. |
||||||
|
* @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be |
||||||
|
* returned. |
||||||
|
* |
||||||
|
* @throws RocksDBException thrown if error happens in underlying |
||||||
|
* native library. |
||||||
|
*/ |
||||||
|
|
||||||
|
public static void loadLatestOptions(String dbPath, Env env, DBOptions dbOptions, |
||||||
|
List<ColumnFamilyDescriptor> cfDescs) throws RocksDBException { |
||||||
|
loadLatestOptions(dbPath, env, dbOptions, cfDescs, false); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param dbPath the path to the RocksDB. |
||||||
|
* @param env {@link org.rocksdb.Env} instance. |
||||||
|
* @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be |
||||||
|
* filled and returned. |
||||||
|
* @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be |
||||||
|
* returned. |
||||||
|
* @param ignoreUnknownOptions this flag can be set to true if you want to |
||||||
|
* ignore options that are from a newer version of the db, esentially for |
||||||
|
* forward compatibility. |
||||||
|
* |
||||||
|
* @throws RocksDBException thrown if error happens in underlying |
||||||
|
* native library. |
||||||
|
*/ |
||||||
|
public static void loadLatestOptions(String dbPath, Env env, DBOptions dbOptions, |
||||||
|
List<ColumnFamilyDescriptor> cfDescs, boolean ignoreUnknownOptions) throws RocksDBException { |
||||||
|
loadLatestOptions( |
||||||
|
dbPath, env.nativeHandle_, dbOptions.nativeHandle_, cfDescs, ignoreUnknownOptions); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Similar to LoadLatestOptions, this function constructs the DBOptions |
||||||
|
* and ColumnFamilyDescriptors based on the specified RocksDB Options file. |
||||||
|
* See LoadLatestOptions above. |
||||||
|
* |
||||||
|
* @param optionsFileName the RocksDB options file path. |
||||||
|
* @param env {@link org.rocksdb.Env} instance. |
||||||
|
* @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be |
||||||
|
* filled and returned. |
||||||
|
* @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be |
||||||
|
* returned. |
||||||
|
* |
||||||
|
* @throws RocksDBException thrown if error happens in underlying |
||||||
|
* native library. |
||||||
|
*/ |
||||||
|
public static void loadOptionsFromFile(String optionsFileName, Env env, DBOptions dbOptions, |
||||||
|
List<ColumnFamilyDescriptor> cfDescs) throws RocksDBException { |
||||||
|
loadOptionsFromFile(optionsFileName, env, dbOptions, cfDescs, false); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @param optionsFileName the RocksDB options file path. |
||||||
|
* @param env {@link org.rocksdb.Env} instance. |
||||||
|
* @param dbOptions {@link org.rocksdb.DBOptions} instance. This will be |
||||||
|
* filled and returned. |
||||||
|
* @param cfDescs A list of {@link org.rocksdb.ColumnFamilyDescriptor}'s be |
||||||
|
* returned. |
||||||
|
* @param ignoreUnknownOptions this flag can be set to true if you want to |
||||||
|
* ignore options that are from a newer version of the db, esentially for |
||||||
|
* forward compatibility. |
||||||
|
* |
||||||
|
* @throws RocksDBException thrown if error happens in underlying |
||||||
|
* native library. |
||||||
|
*/ |
||||||
|
public static void loadOptionsFromFile(String optionsFileName, Env env, DBOptions dbOptions, |
||||||
|
List<ColumnFamilyDescriptor> cfDescs, boolean ignoreUnknownOptions) throws RocksDBException { |
||||||
|
loadOptionsFromFile( |
||||||
|
optionsFileName, env.nativeHandle_, dbOptions.nativeHandle_, cfDescs, ignoreUnknownOptions); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns the latest options file name under the specified RocksDB path. |
||||||
|
* |
||||||
|
* @param dbPath the path to the RocksDB. |
||||||
|
* @param env {@link org.rocksdb.Env} instance. |
||||||
|
* @return the latest options file name under the db path. |
||||||
|
* |
||||||
|
* @throws RocksDBException thrown if error happens in underlying |
||||||
|
* native library. |
||||||
|
*/ |
||||||
|
public static String getLatestOptionsFileName(String dbPath, Env env) throws RocksDBException { |
||||||
|
return getLatestOptionsFileName(dbPath, env.nativeHandle_); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Private constructor. |
||||||
|
* This class has only static methods and shouldn't be instantiated. |
||||||
|
*/ |
||||||
|
private OptionsUtil() {} |
||||||
|
|
||||||
|
// native methods
|
||||||
|
private native static void loadLatestOptions(String dbPath, long envHandle, long dbOptionsHandle, |
||||||
|
List<ColumnFamilyDescriptor> cfDescs, boolean ignoreUnknownOptions) throws RocksDBException; |
||||||
|
private native static void loadOptionsFromFile(String optionsFileName, long envHandle, |
||||||
|
long dbOptionsHandle, List<ColumnFamilyDescriptor> cfDescs, boolean ignoreUnknownOptions) |
||||||
|
throws RocksDBException; |
||||||
|
private native static String getLatestOptionsFileName(String dbPath, long envHandle) |
||||||
|
throws RocksDBException; |
||||||
|
} |
||||||
@ -0,0 +1,126 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb; |
||||||
|
|
||||||
|
import org.junit.ClassRule; |
||||||
|
import org.junit.Rule; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.rules.TemporaryFolder; |
||||||
|
|
||||||
|
import java.util.*; |
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat; |
||||||
|
|
||||||
|
public class OptionsUtilTest { |
||||||
|
@ClassRule |
||||||
|
public static final RocksMemoryResource rocksMemoryResource = new RocksMemoryResource(); |
||||||
|
|
||||||
|
@Rule public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||||
|
|
||||||
|
enum TestAPI { LOAD_LATEST_OPTIONS, LOAD_OPTIONS_FROM_FILE } |
||||||
|
|
||||||
|
@Test |
||||||
|
public void loadLatestOptions() throws RocksDBException { |
||||||
|
verifyOptions(TestAPI.LOAD_LATEST_OPTIONS); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void loadOptionsFromFile() throws RocksDBException { |
||||||
|
verifyOptions(TestAPI.LOAD_OPTIONS_FROM_FILE); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void getLatestOptionsFileName() throws RocksDBException { |
||||||
|
final String dbPath = dbFolder.getRoot().getAbsolutePath(); |
||||||
|
try (final Options options = new Options().setCreateIfMissing(true); |
||||||
|
final RocksDB db = RocksDB.open(options, dbPath)) { |
||||||
|
assertThat(db).isNotNull(); |
||||||
|
} |
||||||
|
|
||||||
|
String fName = OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); |
||||||
|
assertThat(fName).isNotNull(); |
||||||
|
assert(fName.startsWith("OPTIONS-") == true); |
||||||
|
// System.out.println("latest options fileName: " + fName);
|
||||||
|
} |
||||||
|
|
||||||
|
private void verifyOptions(TestAPI apiType) throws RocksDBException { |
||||||
|
final String dbPath = dbFolder.getRoot().getAbsolutePath(); |
||||||
|
final Options options = new Options() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setParanoidChecks(false) |
||||||
|
.setMaxOpenFiles(478) |
||||||
|
.setDelayedWriteRate(1234567L); |
||||||
|
final ColumnFamilyOptions baseDefaultCFOpts = new ColumnFamilyOptions(); |
||||||
|
final byte[] secondCFName = "new_cf".getBytes(); |
||||||
|
final ColumnFamilyOptions baseSecondCFOpts = |
||||||
|
new ColumnFamilyOptions() |
||||||
|
.setWriteBufferSize(70 * 1024) |
||||||
|
.setMaxWriteBufferNumber(7) |
||||||
|
.setMaxBytesForLevelBase(53 * 1024 * 1024) |
||||||
|
.setLevel0FileNumCompactionTrigger(3) |
||||||
|
.setLevel0SlowdownWritesTrigger(51) |
||||||
|
.setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); |
||||||
|
|
||||||
|
// Create a database with a new column family
|
||||||
|
try (final RocksDB db = RocksDB.open(options, dbPath)) { |
||||||
|
assertThat(db).isNotNull(); |
||||||
|
|
||||||
|
// create column family
|
||||||
|
try (final ColumnFamilyHandle columnFamilyHandle = |
||||||
|
db.createColumnFamily(new ColumnFamilyDescriptor(secondCFName, baseSecondCFOpts))) { |
||||||
|
assert(columnFamilyHandle != null); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Read the options back and verify
|
||||||
|
DBOptions dbOptions = new DBOptions(); |
||||||
|
final List<ColumnFamilyDescriptor> cfDescs = new ArrayList<>(); |
||||||
|
String path = dbPath; |
||||||
|
if (apiType == TestAPI.LOAD_LATEST_OPTIONS) { |
||||||
|
OptionsUtil.loadLatestOptions(path, Env.getDefault(), dbOptions, cfDescs, false); |
||||||
|
} else if (apiType == TestAPI.LOAD_OPTIONS_FROM_FILE) { |
||||||
|
path = dbPath + "/" + OptionsUtil.getLatestOptionsFileName(dbPath, Env.getDefault()); |
||||||
|
OptionsUtil.loadOptionsFromFile(path, Env.getDefault(), dbOptions, cfDescs, false); |
||||||
|
} |
||||||
|
|
||||||
|
assertThat(dbOptions.createIfMissing()).isEqualTo(options.createIfMissing()); |
||||||
|
assertThat(dbOptions.paranoidChecks()).isEqualTo(options.paranoidChecks()); |
||||||
|
assertThat(dbOptions.maxOpenFiles()).isEqualTo(options.maxOpenFiles()); |
||||||
|
assertThat(dbOptions.delayedWriteRate()).isEqualTo(options.delayedWriteRate()); |
||||||
|
|
||||||
|
assertThat(cfDescs.size()).isEqualTo(2); |
||||||
|
assertThat(cfDescs.get(0)).isNotNull(); |
||||||
|
assertThat(cfDescs.get(1)).isNotNull(); |
||||||
|
assertThat(cfDescs.get(0).columnFamilyName()).isEqualTo(RocksDB.DEFAULT_COLUMN_FAMILY); |
||||||
|
assertThat(cfDescs.get(1).columnFamilyName()).isEqualTo(secondCFName); |
||||||
|
|
||||||
|
ColumnFamilyOptions defaultCFOpts = cfDescs.get(0).columnFamilyOptions(); |
||||||
|
assertThat(defaultCFOpts.writeBufferSize()).isEqualTo(baseDefaultCFOpts.writeBufferSize()); |
||||||
|
assertThat(defaultCFOpts.maxWriteBufferNumber()) |
||||||
|
.isEqualTo(baseDefaultCFOpts.maxWriteBufferNumber()); |
||||||
|
assertThat(defaultCFOpts.maxBytesForLevelBase()) |
||||||
|
.isEqualTo(baseDefaultCFOpts.maxBytesForLevelBase()); |
||||||
|
assertThat(defaultCFOpts.level0FileNumCompactionTrigger()) |
||||||
|
.isEqualTo(baseDefaultCFOpts.level0FileNumCompactionTrigger()); |
||||||
|
assertThat(defaultCFOpts.level0SlowdownWritesTrigger()) |
||||||
|
.isEqualTo(baseDefaultCFOpts.level0SlowdownWritesTrigger()); |
||||||
|
assertThat(defaultCFOpts.bottommostCompressionType()) |
||||||
|
.isEqualTo(baseDefaultCFOpts.bottommostCompressionType()); |
||||||
|
|
||||||
|
ColumnFamilyOptions secondCFOpts = cfDescs.get(1).columnFamilyOptions(); |
||||||
|
assertThat(secondCFOpts.writeBufferSize()).isEqualTo(baseSecondCFOpts.writeBufferSize()); |
||||||
|
assertThat(secondCFOpts.maxWriteBufferNumber()) |
||||||
|
.isEqualTo(baseSecondCFOpts.maxWriteBufferNumber()); |
||||||
|
assertThat(secondCFOpts.maxBytesForLevelBase()) |
||||||
|
.isEqualTo(baseSecondCFOpts.maxBytesForLevelBase()); |
||||||
|
assertThat(secondCFOpts.level0FileNumCompactionTrigger()) |
||||||
|
.isEqualTo(baseSecondCFOpts.level0FileNumCompactionTrigger()); |
||||||
|
assertThat(secondCFOpts.level0SlowdownWritesTrigger()) |
||||||
|
.isEqualTo(baseSecondCFOpts.level0SlowdownWritesTrigger()); |
||||||
|
assertThat(secondCFOpts.bottommostCompressionType()) |
||||||
|
.isEqualTo(baseSecondCFOpts.bottommostCompressionType()); |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue