Merge pull request #157 from ankgup87/master

[Java] Add RestoreBackupableDB

java/Makefile

@@ -1,4 +1,5 @@
-NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter
+NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB
+
 NATIVE_INCLUDE = ./include
 ROCKSDB_JAR = rocksdbjni.jar
 

java/org/rocksdb/BackupableDBOptions.java

@@ -11,11 +11,39 @@ package org.rocksdb;
  *
  * Note that dispose() must be called before an Options instance
  * become out-of-scope to release the allocated memory in c++.
+ *
+ * @param path Where to keep the backup files. Has to be different than dbname.
+       Best to set this to dbname_ + "/backups"
+ * @param shareTableFiles If share_table_files == true, backup will assume that
+ *     table files with same name have the same contents. This enables
+ *     incremental backups and avoids unnecessary data copies. If
+ *     share_table_files == false, each backup will be on its own and will not
+ *     share any data with other backups. default: true
+ * @param sync If sync == true, we can guarantee you'll get consistent backup
+ *     even on a machine crash/reboot. Backup process is slower with sync
+ *     enabled. If sync == false, we don't guarantee anything on machine reboot.
+ *     However, chances are some of the backups are consistent. Default: true
+ * @param destroyOldData If true, it will delete whatever backups there are
+ *     already. Default: false
+ * @param backupLogFiles If false, we won't backup log files. This option can be
+ *     useful for backing up in-memory databases where log file are persisted,
+ *     but table files are in memory. Default: true
+ * @param backupRateLimit Max bytes that can be transferred in a second during
+ *     backup. If 0 or negative, then go as fast as you can. Default: 0
+ * @param restoreRateLimit Max bytes that can be transferred in a second during
+ *     restore. If 0 or negative, then go as fast as you can. Default: 0
  */
 public class BackupableDBOptions extends RocksObject {
-  public BackupableDBOptions(String path) {
+  public BackupableDBOptions(String path, boolean shareTableFiles, boolean sync,
+      boolean destroyOldData, boolean backupLogFiles, long backupRateLimit,
+      long restoreRateLimit) {
     super();
-    newBackupableDBOptions(path);
+    
+    backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit;
+    restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit;
+    
+    newBackupableDBOptions(path, shareTableFiles, sync, destroyOldData,
+        backupLogFiles, backupRateLimit, restoreRateLimit);
   }
 
   /**
@@ -37,7 +65,9 @@ public class BackupableDBOptions extends RocksObject {
     disposeInternal(nativeHandle_);
   }
 
-  private native void newBackupableDBOptions(String path);
+  private native void newBackupableDBOptions(String path,
+      boolean shareTableFiles, boolean sync, boolean destroyOldData,
+      boolean backupLogFiles, long backupRateLimit, long restoreRateLimit);
   private native String backupDir(long handle);
   private native void disposeInternal(long handle);
 }

java/org/rocksdb/RestoreBackupableDB.java

@@ -0,0 +1,84 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * This class is used to access information about backups and restore from them.
+ *
+ * Note that dispose() must be called before this instance become out-of-scope
+ * to release the allocated memory in c++.
+ *
+ * @param options Instance of BackupableDBOptions.
+ */
+public class RestoreBackupableDB extends RocksObject {
+  public RestoreBackupableDB(BackupableDBOptions options) {
+    super();
+    nativeHandle_ = newRestoreBackupableDB(options.nativeHandle_);
+  }
+
+  /**
+   * Restore from backup with backup_id
+   * IMPORTANT -- if options_.share_table_files == true and you restore DB
+   * from some backup that is not the latest, and you start creating new
+   * backups from the new DB, they will probably fail.
+   *
+   * Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3.
+   * If you add new data to the DB and try creating a new backup now, the
+   * database will diverge from backups 4 and 5 and the new backup will fail.
+   * If you want to create new backup, you will first have to delete backups 4
+   * and 5.
+   */
+  public void restoreDBFromBackup(long backupId, String dbDir, String walDir,
+      RestoreOptions restoreOptions) throws RocksDBException {
+    restoreDBFromBackup0(nativeHandle_, backupId, dbDir, walDir,
+        restoreOptions.nativeHandle_);
+  }
+
+  /**
+   * Restore from the latest backup.
+   */
+  public void restoreDBFromLatestBackup(String dbDir, String walDir,
+      RestoreOptions restoreOptions) throws RocksDBException {
+    restoreDBFromLatestBackup0(nativeHandle_, dbDir, walDir,
+        restoreOptions.nativeHandle_);
+  }
+
+  /**
+   * Deletes old backups, keeping latest numBackupsToKeep alive.
+   *
+   * @param Number of latest backups to keep
+   */
+  public void purgeOldBackups(int numBackupsToKeep) throws RocksDBException {
+    purgeOldBackups0(nativeHandle_, numBackupsToKeep);
+  }
+
+  /**
+   * Deletes a specific backup.
+   *
+   * @param ID of backup to delete.
+   */
+  public void deleteBackup(long backupId) throws RocksDBException {
+    deleteBackup0(nativeHandle_, backupId);
+  }
+
+  /**
+   * Release the memory allocated for the current instance
+   * in the c++ side.
+   */
+  @Override public synchronized void disposeInternal() {
+    assert(isInitialized());
+    dispose(nativeHandle_);
+  }
+
+  private native long newRestoreBackupableDB(long options);
+  private native void restoreDBFromBackup0(long nativeHandle, long backupId,
+      String dbDir, String walDir, long restoreOptions) throws RocksDBException;
+  private native void restoreDBFromLatestBackup0(long nativeHandle,
+      String dbDir, String walDir, long restoreOptions) throws RocksDBException;
+  private native void purgeOldBackups0(long nativeHandle, int numBackupsToKeep);
+  private native void deleteBackup0(long nativeHandle, long backupId);
+  private native void dispose(long nativeHandle);
+}

java/org/rocksdb/RestoreOptions.java

@@ -0,0 +1,37 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * RestoreOptions to control the behavior of restore.
+ *
+ * Note that dispose() must be called before this instance become out-of-scope
+ * to release the allocated memory in c++.
+ *
+ * @param If true, restore won't overwrite the existing log files in wal_dir. It
+ *     will also move all log files from archive directory to wal_dir. Use this
+ *     option in combination with BackupableDBOptions::backup_log_files = false
+ *     for persisting in-memory databases.
+ *     Default: false
+ */
+public class RestoreOptions extends RocksObject {
+  public RestoreOptions(boolean keepLogFiles) {
+    super();
+    nativeHandle_ = newRestoreOptions(keepLogFiles);
+  }
+
+  /**
+   * Release the memory allocated for the current instance
+   * in the c++ side.
+   */
+  @Override public synchronized void disposeInternal() {
+    assert(isInitialized());
+    dispose(nativeHandle_);
+  }
+
+  private native long newRestoreOptions(boolean keepLogFiles);
+  private native void dispose(long handle);
+}

java/org/rocksdb/test/BackupableDBTest.java

@@ -18,15 +18,37 @@ public class BackupableDBTest {
     Options opt = new Options();
     opt.setCreateIfMissing(true);
 
-    BackupableDBOptions bopt = new BackupableDBOptions(backup_path);
+    BackupableDBOptions bopt = new BackupableDBOptions(backup_path, false,
+        true, false, true, 0, 0);
     BackupableDB bdb = null;
 
     try {
       bdb = BackupableDB.open(opt, bopt, db_path);
-      bdb.put("hello".getBytes(), "BackupableDB".getBytes());
+
+      bdb.put("abc".getBytes(), "def".getBytes());
+      bdb.put("ghi".getBytes(), "jkl".getBytes());
       bdb.createNewBackup(true);
-      byte[] value = bdb.get("hello".getBytes());
+
-      assert(new String(value).equals("BackupableDB"));
+      // delete record after backup
+      bdb.remove("abc".getBytes());
+      byte[] value = bdb.get("abc".getBytes());
+      assert(value == null);
+      bdb.close();
+
+      // restore from backup
+      RestoreOptions ropt = new RestoreOptions(false);
+      RestoreBackupableDB rdb = new RestoreBackupableDB(bopt);
+      rdb.restoreDBFromLatestBackup(db_path, db_path,
+          ropt);
+      rdb.dispose();
+      ropt.dispose();
+
+      // verify that backed up data contains deleted record
+      bdb = BackupableDB.open(opt, bopt, db_path);
+      value = bdb.get("abc".getBytes());
+      assert(new String(value).equals("def"));
+
+      System.out.println("Backup and restore test passed");
     } catch (RocksDBException e) {
       System.err.format("[ERROR]: %s%n", e);
       e.printStackTrace();

java/rocksjni/backupablejni.cc

@@ -4,7 +4,8 @@
 // of patent rights can be found in the PATENTS file in the same directory.
 //
 // This file implements the "bridge" between Java and C++ and enables
-// calling c++ rocksdb::DB methods from Java side.
+// calling c++ rocksdb::BackupableDB and rocksdb::BackupableDBOptions methods
+// from Java side.
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -51,9 +52,18 @@ void Java_org_rocksdb_BackupableDB_createNewBackup(
  * Signature: (Ljava/lang/String;)V
  */
 void Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions(
-    JNIEnv* env, jobject jobj, jstring jpath) {
+    JNIEnv* env, jobject jobj, jstring jpath, jboolean jshare_table_files,
+    jboolean jsync, jboolean jdestroy_old_data, jboolean jbackup_log_files,
+    jlong jbackup_rate_limit, jlong jrestore_rate_limit) {
+  jbackup_rate_limit = (jbackup_rate_limit <= 0) ? 0 : jbackup_rate_limit;
+  jrestore_rate_limit = (jrestore_rate_limit <= 0) ? 0 : jrestore_rate_limit;
+
   const char* cpath = env->GetStringUTFChars(jpath, 0);
-  auto bopt = new rocksdb::BackupableDBOptions(cpath);
+
+  auto bopt = new rocksdb::BackupableDBOptions(cpath, nullptr,
+      jshare_table_files, nullptr, jsync, jdestroy_old_data, jbackup_log_files,
+      jbackup_rate_limit, jrestore_rate_limit);
+
   env->ReleaseStringUTFChars(jpath, cpath);
 
   rocksdb::BackupableDBOptionsJni::setHandle(env, jobj, bopt);

java/rocksjni/portal.h

@@ -217,6 +217,7 @@ class HistogramDataJni {
     return mid;
   }
 };
+
 class BackupableDBOptionsJni {
  public:
   // Get the java class id of org.rocksdb.BackupableDBOptions.

java/rocksjni/restorejni.cc

@@ -0,0 +1,145 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+//
+// This file implements the "bridge" between Java and C++ and enables
+// calling c++ rocksdb::RestoreBackupableDB and rocksdb::RestoreOptions methods
+// from Java side.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <jni.h>
+#include <iostream>
+#include <string>
+
+#include "include/org_rocksdb_RestoreOptions.h"
+#include "include/org_rocksdb_RestoreBackupableDB.h"
+#include "rocksjni/portal.h"
+#include "utilities/backupable_db.h"
+/*
+ * Class:     org_rocksdb_RestoreOptions
+ * Method:    newRestoreOptions
+ * Signature: (Z)J
+ */
+jlong Java_org_rocksdb_RestoreOptions_newRestoreOptions(JNIEnv* env,
+    jobject jobj, jboolean keep_log_files) {
+  auto ropt = new rocksdb::RestoreOptions(keep_log_files);
+  return reinterpret_cast<jlong>(ropt);
+}
+
+/*
+ * Class:     org_rocksdb_RestoreOptions
+ * Method:    dispose
+ * Signature: (J)V
+ */
+void Java_org_rocksdb_RestoreOptions_dispose(JNIEnv* env, jobject jobj,
+    jlong jhandle) {
+  auto ropt = reinterpret_cast<rocksdb::RestoreOptions*>(jhandle);
+  assert(ropt);
+  delete ropt;
+}
+
+/*
+ * Class:     org_rocksdb_RestoreBackupableDB
+ * Method:    newRestoreBackupableDB
+ * Signature: (J)J
+ */
+jlong Java_org_rocksdb_RestoreBackupableDB_newRestoreBackupableDB(JNIEnv* env,
+    jobject jobj, jlong jopt_handle) {
+  auto opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
+  auto rdb = new rocksdb::RestoreBackupableDB(rocksdb::Env::Default(), *opt);
+  return reinterpret_cast<jlong>(rdb);
+}
+
+/*
+ * Class:     org_rocksdb_RestoreBackupableDB
+ * Method:    restoreDBFromBackup0
+ * Signature: (JJLjava/lang/String;Ljava/lang/String;J)V
+ */
+void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromBackup0(JNIEnv* env,
+    jobject jobj, jlong jhandle, jlong jbackup_id, jstring jdb_dir,
+    jstring jwal_dir, jlong jopt_handle) {
+  auto opt = reinterpret_cast<rocksdb::RestoreOptions*>(jopt_handle);
+
+  const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0);
+  const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0);
+
+  auto rdb = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
+  rocksdb::Status s =
+      rdb->RestoreDBFromBackup(jbackup_id, cdb_dir, cwal_dir, *opt);
+
+  env->ReleaseStringUTFChars(jdb_dir, cdb_dir);
+  env->ReleaseStringUTFChars(jwal_dir, cwal_dir);
+
+  if(!s.ok()) {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+  }
+}
+
+/*
+ * Class:     org_rocksdb_RestoreBackupableDB
+ * Method:    restoreDBFromLatestBackup0
+ * Signature: (JLjava/lang/String;Ljava/lang/String;J)V
+ */
+void Java_org_rocksdb_RestoreBackupableDB_restoreDBFromLatestBackup0(
+    JNIEnv* env, jobject jobj, jlong jhandle, jstring jdb_dir, jstring jwal_dir,
+    jlong jopt_handle) {
+  auto opt = reinterpret_cast<rocksdb::RestoreOptions*>(jopt_handle);
+
+  const char* cdb_dir = env->GetStringUTFChars(jdb_dir, 0);
+  const char* cwal_dir = env->GetStringUTFChars(jwal_dir, 0);
+
+  auto rdb = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
+  rocksdb::Status s =
+      rdb->RestoreDBFromLatestBackup(cdb_dir, cwal_dir, *opt);
+
+  env->ReleaseStringUTFChars(jdb_dir, cdb_dir);
+  env->ReleaseStringUTFChars(jwal_dir, cwal_dir);
+
+  if(!s.ok()) {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+  }
+}
+
+/*
+ * Class:     org_rocksdb_RestoreBackupableDB
+ * Method:    purgeOldBackups0
+ * Signature: (JI)V
+ */
+void Java_org_rocksdb_RestoreBackupableDB_purgeOldBackups0(JNIEnv* env,
+    jobject jobj, jlong jhandle, jint jnum_backups_to_keep) {
+  auto rdb = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
+  rocksdb::Status s = rdb->PurgeOldBackups(jnum_backups_to_keep);
+
+  if(!s.ok()) {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+  }
+}
+
+/*
+ * Class:     org_rocksdb_RestoreBackupableDB
+ * Method:    deleteBackup0
+ * Signature: (JJ)V
+ */
+void Java_org_rocksdb_RestoreBackupableDB_deleteBackup0(JNIEnv* env,
+    jobject jobj, jlong jhandle, jlong jbackup_id) {
+  auto rdb = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
+  rocksdb::Status s = rdb->DeleteBackup(jbackup_id);
+
+  if(!s.ok()) {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+  }
+}
+
+/*
+ * Class:     org_rocksdb_RestoreBackupableDB
+ * Method:    dispose
+ * Signature: (J)V
+ */
+void Java_org_rocksdb_RestoreBackupableDB_dispose(JNIEnv* env, jobject jobj,
+    jlong jhandle) {
+  auto ropt = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
+  assert(ropt);
+  delete ropt;
+}

util/hash_cuckoo_rep.cc

@@ -1,4 +1,3 @@
-
 //  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant