[RocksJava] TTL-Support

Summary:
TTLDB Support exposed in Java-API. It is now
possible to open a datbase using the RocksDB time
to live feature.

Test Plan:
make rocksdbjava
make test
mvn -f rocksjni.pom package

@Adam please test mac osx compile

Reviewers: yhchiang, adamretter, ankgup87

Subscribers: dhruba, adam

Differential Revision: https://reviews.facebook.net/D31449

java/org/rocksdb/TtlDB.java

@@ -89,18 +89,40 @@ public class TtlDB extends RocksDB {
     return ttldb;
   }
 
-  //static Status Open(const DBOptions& db_options, const std::string& dbname,
+  /**
-  //                   const std::vector<ColumnFamilyDescriptor>& column_families,
+   * <p>Opens a TtlDB.</p>
-  //                   std::vector<ColumnFamilyHandle*>* handles,
+   *
-  //                   DBWithTTL** dbptr, std::vector<int32_t> ttls,
+   * @param options {@link org.rocksdb.Options} instance.
-  //                   bool read_only = false);
+   * @param db_path path to database.
+   * @param columnFamilyDescriptors list of column family descriptors
+   * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances
+   *     on open.
+   * @param ttlValues time to live values per column family handle
+   * @param readOnly boolean value indicating if database if db is
+   *     opened read-only.
+   *
+   * @return TtlDB instance.
+   *
+   * @throws RocksDBException thrown if an error occurs within the native
+   *     part of the library.
+   * @throws java.lang.IllegalArgumentException when there is not a ttl value
+   *     per given column family handle.
+   */
   public static TtlDB open(DBOptions options, String db_path,
       List<ColumnFamilyDescriptor> columnFamilyDescriptors,
       List<ColumnFamilyHandle> columnFamilyHandles,
-      List<Integer> ttlValues, boolean readOnly){
+      List<Integer> ttlValues, boolean readOnly) throws RocksDBException {
-
+    if (columnFamilyDescriptors.size() != ttlValues.size()) {
-
+      throw new IllegalArgumentException("There must be a ttl value per column" +
-    return null;
+          "family handle.");
+    }
+    TtlDB ttlDB = new TtlDB();
+    List<Long> cfReferences = ttlDB.open(options.nativeHandle_, db_path, columnFamilyDescriptors,
+        columnFamilyDescriptors.size(),  ttlValues, readOnly);
+    for (int i=0; i<columnFamilyDescriptors.size(); i++) {
+      columnFamilyHandles.add(new ColumnFamilyHandle(ttlDB, cfReferences.get(i)));
+    }
+    return ttlDB;
   }
 
   /**
@@ -146,10 +168,10 @@ public class TtlDB extends RocksDB {
   /**
    * <p>A protected constructor that will be used in the static
    * factory method
-   * {@link #open(DBOptions, String, java.util.List, java.util.List)}
+   * {@link #open(Options, String, int, boolean)}
    * and
-   * {@link #open(DBOptions, String, java.util.List,
+   * {@link #open(DBOptions, String, java.util.List, java.util.List,
-   * java.util.List, java.util.List, boolean)}.
+   * java.util.List, boolean)}.
    * </p>
    */
   protected TtlDB() {
@@ -163,6 +185,10 @@ public class TtlDB extends RocksDB {
 
   private native void open(long optionsHandle, String db_path, int ttl,
       boolean readOnly) throws RocksDBException;
+  private native List<Long> open(long optionsHandle, String db_path,
+      List<ColumnFamilyDescriptor> columnFamilyDescriptors,
+      int columnFamilyDescriptorsLength, List<Integer> ttlValues,
+      boolean readOnly) throws RocksDBException;
   private native long createColumnFamilyWithTtl(long handle,
       ColumnFamilyDescriptor columnFamilyDescriptor, int ttl)
       throws RocksDBException;

java/org/rocksdb/test/TtlDBTest.java

@@ -1,113 +1,169 @@
-// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-// This source code is licensed under the BSD-style license found in the
+// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
+// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
+// of patent rights can be found in the PATENTS file in the same directory.
-
+
-package org.rocksdb.test;
+package org.rocksdb.test;
-
+
-import org.junit.ClassRule;
+import org.junit.ClassRule;
-import org.junit.Rule;
+import org.junit.Rule;
-import org.junit.Test;
+import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TemporaryFolder;
-import org.rocksdb.*;
+import org.rocksdb.*;
-
+
-import java.util.concurrent.TimeUnit;
+import java.util.ArrayList;
-
+import java.util.List;
-import static org.assertj.core.api.Assertions.assertThat;
+import java.util.concurrent.TimeUnit;
-
+
-public class TtlDBTest {
+import static org.assertj.core.api.Assertions.assertThat;
-
+
-  @ClassRule
+public class TtlDBTest {
-  public static final RocksMemoryResource rocksMemoryResource =
+
-      new RocksMemoryResource();
+  @ClassRule
-
+  public static final RocksMemoryResource rocksMemoryResource =
-  @Rule
+      new RocksMemoryResource();
-  public TemporaryFolder dbFolder = new TemporaryFolder();
+
-
+  @Rule
-  @Test
+  public TemporaryFolder dbFolder = new TemporaryFolder();
-  public void ttlDBOpen() throws RocksDBException,
+
-      InterruptedException {
+  @Test
-    Options options = null;
+  public void ttlDBOpen() throws RocksDBException,
-    TtlDB ttlDB = null;
+      InterruptedException {
-    try {
+    Options options = null;
-      options = new Options().
+    TtlDB ttlDB = null;
-          setCreateIfMissing(true).
+    try {
-          setMaxGrandparentOverlapFactor(0).
+      options = new Options().
-          setMaxMemCompactionLevel(0);
+          setCreateIfMissing(true).
-      ttlDB = TtlDB.open(options,
+          setMaxGrandparentOverlapFactor(0).
-          dbFolder.getRoot().getAbsolutePath());
+          setMaxMemCompactionLevel(0);
-      ttlDB.put("key".getBytes(), "value".getBytes());
+      ttlDB = TtlDB.open(options,
-      assertThat(ttlDB.get("key".getBytes())).
+          dbFolder.getRoot().getAbsolutePath());
-          isEqualTo("value".getBytes());
+      ttlDB.put("key".getBytes(), "value".getBytes());
-      assertThat(ttlDB.get("key".getBytes())).isNotNull();
+      assertThat(ttlDB.get("key".getBytes())).
-    } finally {
+          isEqualTo("value".getBytes());
-      if (ttlDB != null) {
+      assertThat(ttlDB.get("key".getBytes())).isNotNull();
-        ttlDB.close();
+    } finally {
-      }
+      if (ttlDB != null) {
-      if (options != null) {
+        ttlDB.close();
-        options.dispose();
+      }
-      }
+      if (options != null) {
-    }
+        options.dispose();
-  }
+      }
-
+    }
-  @Test
+  }
-  public void ttlDBOpenWithTtl() throws RocksDBException,
+
-      InterruptedException {
+  @Test
-    Options options = null;
+  public void ttlDBOpenWithTtl() throws RocksDBException,
-    TtlDB ttlDB = null;
+      InterruptedException {
-    try {
+    Options options = null;
-      options = new Options().
+    TtlDB ttlDB = null;
-          setCreateIfMissing(true).
+    try {
-          setMaxGrandparentOverlapFactor(0).
+      options = new Options().
-          setMaxMemCompactionLevel(0);
+          setCreateIfMissing(true).
-      ttlDB = TtlDB.open(options, dbFolder.getRoot().getAbsolutePath(),
+          setMaxGrandparentOverlapFactor(0).
-          1, false);
+          setMaxMemCompactionLevel(0);
-      ttlDB.put("key".getBytes(), "value".getBytes());
+      ttlDB = TtlDB.open(options, dbFolder.getRoot().getAbsolutePath(),
-      assertThat(ttlDB.get("key".getBytes())).
+          1, false);
-          isEqualTo("value".getBytes());
+      ttlDB.put("key".getBytes(), "value".getBytes());
-      TimeUnit.SECONDS.sleep(2);
+      assertThat(ttlDB.get("key".getBytes())).
-
+          isEqualTo("value".getBytes());
-      ttlDB.compactRange();
+      TimeUnit.SECONDS.sleep(2);
-      assertThat(ttlDB.get("key".getBytes())).isNull();
+
-    } finally {
+      ttlDB.compactRange();
-      if (ttlDB != null) {
+      assertThat(ttlDB.get("key".getBytes())).isNull();
-        ttlDB.close();
+    } finally {
-      }
+      if (ttlDB != null) {
-      if (options != null) {
+        ttlDB.close();
-        options.dispose();
+      }
-      }
+      if (options != null) {
-    }
+        options.dispose();
-  }
+      }
-
+    }
-  @Test
+  }
-  public void createTtlColumnFamily() throws RocksDBException,
+
-      InterruptedException {
+  @Test
-    Options options = null;
+  public void ttlDbOpenWithColumnFamilies() throws RocksDBException, InterruptedException {
-    TtlDB ttlDB = null;
+    DBOptions dbOptions = null;
-    ColumnFamilyHandle columnFamilyHandle = null;
+    TtlDB ttlDB = null;
-    try {
+    List<ColumnFamilyDescriptor> cfNames =
-      options = new Options().setCreateIfMissing(true);
+        new ArrayList<>();
-      ttlDB = TtlDB.open(options,
+    List<ColumnFamilyHandle> columnFamilyHandleList =
-          dbFolder.getRoot().getAbsolutePath());
+        new ArrayList<>();
-      columnFamilyHandle = ttlDB.createColumnFamilyWithTtl(
+    cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
-          new ColumnFamilyDescriptor("new_cf"), 1);
+    cfNames.add(new ColumnFamilyDescriptor("new_cf".getBytes()));
-      ttlDB.put(columnFamilyHandle, "key".getBytes(),
+    List<Integer> ttlValues = new ArrayList<>();
-          "value".getBytes());
+    // Default column family with infinite lifetime
-      assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())).
+    ttlValues.add(0);
-          isEqualTo("value".getBytes());
+    // new column family with 1 second ttl
-      Thread.sleep(2500);
+    ttlValues.add(1);
-      ttlDB.compactRange(columnFamilyHandle);
+
-      assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())).isNull();
+    try {
-    } finally {
+      dbOptions = new DBOptions().
-      if (columnFamilyHandle != null) {
+          setCreateMissingColumnFamilies(true).
-        columnFamilyHandle.dispose();
+          setCreateIfMissing(true);
-      }
+      ttlDB = TtlDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(),
-      if (ttlDB != null) {
+          cfNames, columnFamilyHandleList, ttlValues, false);
-        ttlDB.close();
+
-      }
+      ttlDB.put("key".getBytes(), "value".getBytes());
-      if (options != null) {
+      assertThat(ttlDB.get("key".getBytes())).
-        options.dispose();
+          isEqualTo("value".getBytes());
-      }
+      ttlDB.put(columnFamilyHandleList.get(1), "key".getBytes(),
-    }
+          "value".getBytes());
-  }
+      assertThat(ttlDB.get(columnFamilyHandleList.get(1),
-}
+          "key".getBytes())).isEqualTo("value".getBytes());
+      TimeUnit.SECONDS.sleep(2);
+
+      ttlDB.compactRange();
+      ttlDB.compactRange(columnFamilyHandleList.get(1));
+
+      assertThat(ttlDB.get("key".getBytes())).isNotNull();
+      assertThat(ttlDB.get(columnFamilyHandleList.get(1),
+          "key".getBytes())).isNull();
+
+
+    } finally {
+      for (ColumnFamilyHandle columnFamilyHandle :
+          columnFamilyHandleList) {
+        columnFamilyHandle.dispose();
+      }
+      if (ttlDB != null) {
+        ttlDB.close();
+      }
+      if (dbOptions != null) {
+        dbOptions.dispose();
+      }
+    }
+  }
+
+  @Test
+  public void createTtlColumnFamily() throws RocksDBException,
+      InterruptedException {
+    Options options = null;
+    TtlDB ttlDB = null;
+    ColumnFamilyHandle columnFamilyHandle = null;
+    try {
+      options = new Options().setCreateIfMissing(true);
+      ttlDB = TtlDB.open(options,
+          dbFolder.getRoot().getAbsolutePath());
+      columnFamilyHandle = ttlDB.createColumnFamilyWithTtl(
+          new ColumnFamilyDescriptor("new_cf".getBytes()), 1);
+      ttlDB.put(columnFamilyHandle, "key".getBytes(),
+          "value".getBytes());
+      assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())).
+          isEqualTo("value".getBytes());
+      TimeUnit.SECONDS.sleep(2);
+      ttlDB.compactRange(columnFamilyHandle);
+      assertThat(ttlDB.get(columnFamilyHandle, "key".getBytes())).isNull();
+    } finally {
+      if (columnFamilyHandle != null) {
+        columnFamilyHandle.dispose();
+      }
+      if (ttlDB != null) {
+        ttlDB.close();
+      }
+      if (options != null) {
+        options.dispose();
+      }
+    }
+  }
+}

java/rocksjni/ttl.cc

@@ -7,22 +7,22 @@
 // calling c++ rocksdb::TtlDB methods.
 // from Java side.
 
+#include <jni.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <jni.h>
 #include <string>
 #include <vector>
 
 #include "include/org_rocksdb_TtlDB.h"
-#include "rocksjni/portal.h"
 #include "rocksdb/utilities/db_ttl.h"
+#include "rocksjni/portal.h"
 
 /*
  * Class:     org_rocksdb_TtlDB
  * Method:    open
- * Signature: (JJ)V
+ * Signature: (JLjava/lang/String;IZ)V
  */
-void Java_org_rocksdb_TtlDB_open(JNIEnv* env, jobject jttldb,
+void Java_org_rocksdb_TtlDB_open__JLjava_lang_String_2IZ(JNIEnv* env, jobject jttldb,
     jlong joptions_handle, jstring jdb_path, jint jttl,
     jboolean jread_only) {
   auto opt = reinterpret_cast<rocksdb::Options*>(joptions_handle);
@@ -41,6 +41,107 @@ void Java_org_rocksdb_TtlDB_open(JNIEnv* env, jobject jttldb,
   rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
 }
 
+/*
+ * Class:     org_rocksdb_TtlDB
+ * Method:    open
+ * Signature: (JLjava/lang/String;Ljava/util/List;ILjava/util/List;Z)Ljava/util/List;
+ */
+jobject
+    Java_org_rocksdb_TtlDB_open__JLjava_lang_String_2Ljava_util_List_2ILjava_util_List_2Z(
+    JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path,
+    jobject jcfdesc_list, jint jcfdesc_count, jobject jttl_list,
+    jboolean jread_only) {
+  auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
+  rocksdb::DBWithTTL* db = nullptr;
+  const char* db_path = env->GetStringUTFChars(jdb_path, 0);
+
+  std::vector<jbyte*> cfnames_to_free;
+  std::vector<jbyteArray> jcfnames_for_free;
+
+  std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
+  std::vector<int32_t> ttl_values;
+  std::vector<rocksdb::ColumnFamilyHandle* > handles;
+  // get iterator for ColumnFamilyDescriptors
+  jobject iteratorObj = env->CallObjectMethod(
+      jcfdesc_list, rocksdb::ListJni::getIteratorMethod(env));
+
+  // iterate over ColumnFamilyDescriptors
+  while (env->CallBooleanMethod(
+      iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
+      // get ColumnFamilyDescriptor
+      jobject jcf_descriptor = env->CallObjectMethod(iteratorObj,
+          rocksdb::ListJni::getNextMethod(env));
+      // get ColumnFamilyName
+      jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
+          jcf_descriptor,
+          rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
+              env)));
+      // get CF Options
+      jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
+          rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
+              env));
+      rocksdb::ColumnFamilyOptions* cfOptions =
+          rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
+
+      jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
+      // free allocated cfnames after call to open
+      cfnames_to_free.push_back(cfname);
+      jcfnames_for_free.push_back(byteArray);
+      column_families.push_back(rocksdb::ColumnFamilyDescriptor(
+          reinterpret_cast<const char *>(cfname),
+          *cfOptions));
+  }
+  // get iterator for TTL values
+  iteratorObj = env->CallObjectMethod(
+        jttl_list, rocksdb::ListJni::getIteratorMethod(env));
+  // iterate over TTL values
+  while (env->CallBooleanMethod(
+      iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
+     // get TTL object
+     jobject jttl_object = env->CallObjectMethod(iteratorObj,
+       rocksdb::ListJni::getNextMethod(env));
+     // get Integer value
+     jclass jIntClazz = env->FindClass("java/lang/Integer");
+     jmethodID getVal = env->GetMethodID(jIntClazz, "intValue", "()I");
+     ttl_values.push_back(env->CallIntMethod(jttl_object, getVal));
+  }
+  rocksdb::Status s = rocksdb::DBWithTTL::Open(*opt, db_path, column_families,
+      &handles, &db, ttl_values, jread_only);
+
+  env->ReleaseStringUTFChars(jdb_path, db_path);
+  // free jbyte allocations
+  for (std::vector<jbyte*>::size_type i = 0;
+      i != cfnames_to_free.size(); i++) {
+    // free  cfnames
+    env->ReleaseByteArrayElements(jcfnames_for_free[i], cfnames_to_free[i], 0);
+  }
+
+  // check if open operation was successful
+  if (s.ok()) {
+    rocksdb::RocksDBJni::setHandle(env, jdb, db);
+    jclass jListClazz = env->FindClass("java/util/ArrayList");
+    jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
+        env, jListClazz);
+    jobject jcfhandle_list = env->NewObject(jListClazz,
+        midList, handles.size());
+    // insert in java list
+    for (std::vector<rocksdb::ColumnFamilyHandle*>::size_type i = 0;
+        i != handles.size(); i++) {
+      // jlong must be converted to Long due to collections restrictions
+      jclass jLongClazz = env->FindClass("java/lang/Long");
+      jmethodID midLong = env->GetMethodID(jLongClazz, "<init>", "(J)V");
+      jobject obj = env->NewObject(jLongClazz, midLong,
+          reinterpret_cast<jlong>(handles[i]));
+      env->CallBooleanMethod(jcfhandle_list,
+          rocksdb::ListJni::getListAddMethodId(env), obj);
+    }
+
+    return jcfhandle_list;
+  }
+  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+  return nullptr;
+}
+
 /*
  * Class:     org_rocksdb_TtlDB
  * Method:    createColumnFamilyWithTtl
@@ -52,9 +153,11 @@ jlong Java_org_rocksdb_TtlDB_createColumnFamilyWithTtl(
   rocksdb::ColumnFamilyHandle* handle;
   auto db_handle = reinterpret_cast<rocksdb::DBWithTTL*>(jdb_handle);
 
-  jstring jstr = (jstring) env->CallObjectMethod(jcf_descriptor,
+  // get ColumnFamilyName
+  jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
+      jcf_descriptor,
       rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
-      env));
+          env)));
   // get CF Options
   jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
       rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
@@ -62,10 +165,10 @@ jlong Java_org_rocksdb_TtlDB_createColumnFamilyWithTtl(
   rocksdb::ColumnFamilyOptions* cfOptions =
       rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
 
-  const char* cfname = env->GetStringUTFChars(jstr, 0);
+  jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
   rocksdb::Status s = db_handle->CreateColumnFamilyWithTtl(
-      *cfOptions, cfname, &handle, jttl);
+      *cfOptions, reinterpret_cast<char *>(cfname), &handle, jttl);
-  env->ReleaseStringUTFChars(jstr, cfname);
+  env->ReleaseByteArrayElements(byteArray, cfname, 0);
 
   if (s.ok()) {
     return reinterpret_cast<jlong>(handle);