diff --git a/java/Makefile b/java/Makefile index feacb8346..d124689f6 100644 --- a/java/Makefile +++ b/java/Makefile @@ -1,4 +1,5 @@ -NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter +NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB + NATIVE_INCLUDE = ./include ROCKSDB_JAR = rocksdbjni.jar diff --git a/java/org/rocksdb/BackupableDBOptions.java b/java/org/rocksdb/BackupableDBOptions.java index f229d88aa..c72b44f8c 100644 --- a/java/org/rocksdb/BackupableDBOptions.java +++ b/java/org/rocksdb/BackupableDBOptions.java @@ -11,11 +11,39 @@ package org.rocksdb; * * Note that dispose() must be called before an Options instance * become out-of-scope to release the allocated memory in c++. + * + * @param path Where to keep the backup files. Has to be different than dbname. + Best to set this to dbname_ + "/backups" + * @param shareTableFiles If share_table_files == true, backup will assume that + * table files with same name have the same contents. This enables + * incremental backups and avoids unnecessary data copies. If + * share_table_files == false, each backup will be on its own and will not + * share any data with other backups. default: true + * @param sync If sync == true, we can guarantee you'll get consistent backup + * even on a machine crash/reboot. Backup process is slower with sync + * enabled. If sync == false, we don't guarantee anything on machine reboot. + * However, chances are some of the backups are consistent. Default: true + * @param destroyOldData If true, it will delete whatever backups there are + * already. Default: false + * @param backupLogFiles If false, we won't backup log files. This option can be + * useful for backing up in-memory databases where log file are persisted, + * but table files are in memory. Default: true + * @param backupRateLimit Max bytes that can be transferred in a second during + * backup. If 0 or negative, then go as fast as you can. Default: 0 + * @param restoreRateLimit Max bytes that can be transferred in a second during + * restore. If 0 or negative, then go as fast as you can. Default: 0 */ public class BackupableDBOptions extends RocksObject { - public BackupableDBOptions(String path) { + public BackupableDBOptions(String path, boolean shareTableFiles, boolean sync, + boolean destroyOldData, boolean backupLogFiles, long backupRateLimit, + long restoreRateLimit) { super(); - newBackupableDBOptions(path); + + backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit; + restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit; + + newBackupableDBOptions(path, shareTableFiles, sync, destroyOldData, + backupLogFiles, backupRateLimit, restoreRateLimit); } /** @@ -37,7 +65,9 @@ public class BackupableDBOptions extends RocksObject { disposeInternal(nativeHandle_); } - private native void newBackupableDBOptions(String path); + private native void newBackupableDBOptions(String path, + boolean shareTableFiles, boolean sync, boolean destroyOldData, + boolean backupLogFiles, long backupRateLimit, long restoreRateLimit); private native String backupDir(long handle); private native void disposeInternal(long handle); } diff --git a/java/org/rocksdb/RestoreBackupableDB.java b/java/org/rocksdb/RestoreBackupableDB.java new file mode 100644 index 000000000..dbde447a0 --- /dev/null +++ b/java/org/rocksdb/RestoreBackupableDB.java @@ -0,0 +1,84 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb; + +/** + * This class is used to access information about backups and restore from them. + * + * Note that dispose() must be called before this instance become out-of-scope + * to release the allocated memory in c++. + * + * @param options Instance of BackupableDBOptions. + */ +public class RestoreBackupableDB extends RocksObject { + public RestoreBackupableDB(BackupableDBOptions options) { + super(); + nativeHandle_ = newRestoreBackupableDB(options.nativeHandle_); + } + + /** + * Restore from backup with backup_id + * IMPORTANT -- if options_.share_table_files == true and you restore DB + * from some backup that is not the latest, and you start creating new + * backups from the new DB, they will probably fail. + * + * Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3. + * If you add new data to the DB and try creating a new backup now, the + * database will diverge from backups 4 and 5 and the new backup will fail. + * If you want to create new backup, you will first have to delete backups 4 + * and 5. + */ + public void restoreDBFromBackup(long backupId, String dbDir, String walDir, + RestoreOptions restoreOptions) throws RocksDBException { + restoreDBFromBackup0(nativeHandle_, backupId, dbDir, walDir, + restoreOptions.nativeHandle_); + } + + /** + * Restore from the latest backup. + */ + public void restoreDBFromLatestBackup(String dbDir, String walDir, + RestoreOptions restoreOptions) throws RocksDBException { + restoreDBFromLatestBackup0(nativeHandle_, dbDir, walDir, + restoreOptions.nativeHandle_); + } + + /** + * Deletes old backups, keeping latest numBackupsToKeep alive. + * + * @param Number of latest backups to keep + */ + public void purgeOldBackups(int numBackupsToKeep) throws RocksDBException { + purgeOldBackups0(nativeHandle_, numBackupsToKeep); + } + + /** + * Deletes a specific backup. + * + * @param ID of backup to delete. + */ + public void deleteBackup(long backupId) throws RocksDBException { + deleteBackup0(nativeHandle_, backupId); + } + + /** + * Release the memory allocated for the current instance + * in the c++ side. + */ + @Override public synchronized void disposeInternal() { + assert(isInitialized()); + dispose(nativeHandle_); + } + + private native long newRestoreBackupableDB(long options); + private native void restoreDBFromBackup0(long nativeHandle, long backupId, + String dbDir, String walDir, long restoreOptions) throws RocksDBException; + private native void restoreDBFromLatestBackup0(long nativeHandle, + String dbDir, String walDir, long restoreOptions) throws RocksDBException; + private native void purgeOldBackups0(long nativeHandle, int numBackupsToKeep); + private native void deleteBackup0(long nativeHandle, long backupId); + private native void dispose(long nativeHandle); +} diff --git a/java/org/rocksdb/RestoreOptions.java b/java/org/rocksdb/RestoreOptions.java new file mode 100644 index 000000000..77a2b99bc --- /dev/null +++ b/java/org/rocksdb/RestoreOptions.java @@ -0,0 +1,37 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +package org.rocksdb; + +/** + * RestoreOptions to control the behavior of restore. + * + * Note that dispose() must be called before this instance become out-of-scope + * to release the allocated memory in c++. + * + * @param If true, restore won't overwrite the existing log files in wal_dir. It + * will also move all log files from archive directory to wal_dir. Use this + * option in combination with BackupableDBOptions::backup_log_files = false + * for persisting in-memory databases. + * Default: false + */ +public class RestoreOptions extends RocksObject { + public RestoreOptions(boolean keepLogFiles) { + super(); + nativeHandle_ = newRestoreOptions(keepLogFiles); + } + + /** + * Release the memory allocated for the current instance + * in the c++ side. + */ + @Override public synchronized void disposeInternal() { + assert(isInitialized()); + dispose(nativeHandle_); + } + + private native long newRestoreOptions(boolean keepLogFiles); + private native void dispose(long handle); +} diff --git a/java/org/rocksdb/test/BackupableDBTest.java b/java/org/rocksdb/test/BackupableDBTest.java index f0fc3d501..9d3a64c06 100644 --- a/java/org/rocksdb/test/BackupableDBTest.java +++ b/java/org/rocksdb/test/BackupableDBTest.java @@ -18,15 +18,37 @@ public class BackupableDBTest { Options opt = new Options(); opt.setCreateIfMissing(true); - BackupableDBOptions bopt = new BackupableDBOptions(backup_path); + BackupableDBOptions bopt = new BackupableDBOptions(backup_path, false, + true, false, true, 0, 0); BackupableDB bdb = null; try { bdb = BackupableDB.open(opt, bopt, db_path); - bdb.put("hello".getBytes(), "BackupableDB".getBytes()); + + bdb.put("abc".getBytes(), "def".getBytes()); + bdb.put("ghi".getBytes(), "jkl".getBytes()); bdb.createNewBackup(true); - byte[] value = bdb.get("hello".getBytes()); - assert(new String(value).equals("BackupableDB")); + + // delete record after backup + bdb.remove("abc".getBytes()); + byte[] value = bdb.get("abc".getBytes()); + assert(value == null); + bdb.close(); + + // restore from backup + RestoreOptions ropt = new RestoreOptions(false); + RestoreBackupableDB rdb = new RestoreBackupableDB(bopt); + rdb.restoreDBFromLatestBackup(db_path, db_path, + ropt); + rdb.dispose(); + ropt.dispose(); + + // verify that backed up data contains deleted record + bdb = BackupableDB.open(opt, bopt, db_path); + value = bdb.get("abc".getBytes()); + assert(new String(value).equals("def")); + + System.out.println("Backup and restore test passed"); } catch (RocksDBException e) { System.err.format("[ERROR]: %s%n", e); e.printStackTrace(); diff --git a/java/rocksjni/backupablejni.cc b/java/rocksjni/backupablejni.cc index 956912ef1..6d822b8b6 100644 --- a/java/rocksjni/backupablejni.cc +++ b/java/rocksjni/backupablejni.cc @@ -4,7 +4,8 @@ // of patent rights can be found in the PATENTS file in the same directory. // // This file implements the "bridge" between Java and C++ and enables -// calling c++ rocksdb::DB methods from Java side. +// calling c++ rocksdb::BackupableDB and rocksdb::BackupableDBOptions methods +// from Java side. #include #include @@ -51,9 +52,18 @@ void Java_org_rocksdb_BackupableDB_createNewBackup( * Signature: (Ljava/lang/String;)V */ void Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions( - JNIEnv* env, jobject jobj, jstring jpath) { + JNIEnv* env, jobject jobj, jstring jpath, jboolean jshare_table_files, + jboolean jsync, jboolean jdestroy_old_data, jboolean jbackup_log_files, + jlong jbackup_rate_limit, jlong jrestore_rate_limit) { + jbackup_rate_limit = (jbackup_rate_limit <= 0) ? 0 : jbackup_rate_limit; + jrestore_rate_limit = (jrestore_rate_limit <= 0) ? 0 : jrestore_rate_limit; + const char* cpath = env->GetStringUTFChars(jpath, 0); - auto bopt = new rocksdb::BackupableDBOptions(cpath); + + auto bopt = new rocksdb::BackupableDBOptions(cpath, nullptr, + jshare_table_files, nullptr, jsync, jdestroy_old_data, jbackup_log_files, + jbackup_rate_limit, jrestore_rate_limit); + env->ReleaseStringUTFChars(jpath, cpath); rocksdb::BackupableDBOptionsJni::setHandle(env, jobj, bopt); diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index bd37bffcf..b4f874330 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -217,6 +217,7 @@ class HistogramDataJni { return mid; } }; + class BackupableDBOptionsJni { public: // Get the java class id of org.rocksdb.BackupableDBOptions. diff --git a/java/rocksjni/restorejni.cc b/java/rocksjni/restorejni.cc new file mode 100644 index 000000000..a6ade81a7 --- /dev/null +++ b/java/rocksjni/restorejni.cc @@ -0,0 +1,145 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// This file implements the "bridge" between Java and C++ and enables +// calling c++ rocksdb::RestoreBackupableDB and rocksdb::RestoreOptions methods +// from Java side. + +#include +#include +#include +#include +#include + +#include "include/org_rocksdb_RestoreOptions.h" +#include "include/org_rocksdb_RestoreBackupableDB.h" +#include "rocksjni/portal.h" +#include "utilities/backupable_db.h" +/* + * Class: org_rocksdb_RestoreOptions + * Method: newRestoreOptions + * Signature: (Z)J + */ +jlong Java_org_rocksdb_RestoreOptions_newRestoreOptions(JNIEnv* env, + jobject jobj, jboolean keep_log_files) { + auto ropt = new rocksdb::RestoreOptions(keep_log_files); + return reinterpret_cast(ropt); +} + +/* + * Class: org_rocksdb_RestoreOptions + * Method: dispose + * Signature: (J)V + */ +void Java_org_rocksdb_RestoreOptions_dispose(JNIEnv* env, jobject jobj, + jlong jhandle) { + auto ropt = reinterpret_cast(jhandle); + assert(ropt); + delete ropt; +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: newRestoreBackupableDB + * Signature: (J)J + */ +jlong Java_org_rocksdb_RestoreBackupableDB_newRestoreBackupableDB(JNIEnv* env, + jobject jobj, jlong jopt_handle) { + auto opt = reinterpret_cast(jopt_handle); + auto rdb = new rocksdb::RestoreBackupableDB(rocksdb::Env::Default(), *opt); + return reinterpret_cast(rdb); +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: restoreDBFromBackup0 + * Signature: (JJLjava/lang/String;Ljava/lang/String;J)V + */ +void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromBackup0(JNIEnv* env, + jobject jobj, jlong jhandle, jlong jbackup_id, jstring jdb_dir, + jstring jwal_dir, jlong jopt_handle) { + auto opt = reinterpret_cast(jopt_handle); + + const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0); + const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0); + + auto rdb = reinterpret_cast(jhandle); + rocksdb::Status s = + rdb->RestoreDBFromBackup(jbackup_id, cdb_dir, cwal_dir, *opt); + + env->ReleaseStringUTFChars(jdb_dir, cdb_dir); + env->ReleaseStringUTFChars(jwal_dir, cwal_dir); + + if(!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: restoreDBFromLatestBackup0 + * Signature: (JLjava/lang/String;Ljava/lang/String;J)V + */ +void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromLatestBackup0( + JNIEnv* env, jobject jobj, jlong jhandle, jstring jdb_dir, jstring jwal_dir, + jlong jopt_handle) { + auto opt = reinterpret_cast(jopt_handle); + + const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0); + const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0); + + auto rdb = reinterpret_cast(jhandle); + rocksdb::Status s = + rdb->RestoreDBFromLatestBackup(cdb_dir, cwal_dir, *opt); + + env->ReleaseStringUTFChars(jdb_dir, cdb_dir); + env->ReleaseStringUTFChars(jwal_dir, cwal_dir); + + if(!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: purgeOldBackups0 + * Signature: (JI)V + */ +void Java_org_rocksdb_RestoreBackupableDB_purgeOldBackups0(JNIEnv* env, + jobject jobj, jlong jhandle, jint jnum_backups_to_keep) { + auto rdb = reinterpret_cast(jhandle); + rocksdb::Status s = rdb->PurgeOldBackups(jnum_backups_to_keep); + + if(!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: deleteBackup0 + * Signature: (JJ)V + */ +void Java_org_rocksdb_RestoreBackupableDB_deleteBackup0(JNIEnv* env, + jobject jobj, jlong jhandle, jlong jbackup_id) { + auto rdb = reinterpret_cast(jhandle); + rocksdb::Status s = rdb->DeleteBackup(jbackup_id); + + if(!s.ok()) { + rocksdb::RocksDBExceptionJni::ThrowNew(env, s); + } +} + +/* + * Class: org_rocksdb_RestoreBackupableDB + * Method: dispose + * Signature: (J)V + */ +void Java_org_rocksdb_RestoreBackupableDB_dispose(JNIEnv* env, jobject jobj, + jlong jhandle) { + auto ropt = reinterpret_cast(jhandle); + assert(ropt); + delete ropt; +} diff --git a/util/hash_cuckoo_rep.cc b/util/hash_cuckoo_rep.cc index e2d2c38e6..e6426a91c 100644 --- a/util/hash_cuckoo_rep.cc +++ b/util/hash_cuckoo_rep.cc @@ -1,4 +1,3 @@ - // Copyright (c) 2014, Facebook, Inc. All rights reserved. // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant