jni: expose memtable_whole_key_filtering option (#9394)

Summary: refer to: https://github.com/facebook/rocksdb/wiki/Prefix-Seek#configure-prefix-bloom-filter Pull Request resolved: https://github.com/facebook/rocksdb/pull/9394 Reviewed By: mrambacher Differential Revision: D33671533 Pulled By: ajkr fbshipit-source-id: d90db1712efdd5dd65020329867381d6b3cf2626
2 years ago · 83ff350ff2
parent afc280fdfd
commit 83ff350ff2
10 changed files with 154 additions and 14 deletions
--- a/include/rocksdb/advanced_options.h
+++ b/include/rocksdb/advanced_options.h
@ -377,7 +377,7 @@ struct AdvancedColumnFamilyOptions {
  //
  // If this value is larger than 0.25, it is sanitized to 0.25.
  //
-  // Default: 0 (disable)
+  // Default: 0 (disabled)
  //
  // Dynamically changeable through SetOptions() API
  double memtable_prefix_bloom_size_ratio = 0.0;
@ -386,7 +386,7 @@ struct AdvancedColumnFamilyOptions {
  // if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering
  // can potentially reduce CPU usage for point-look-ups.
  //
-  // Default: false (disable)
+  // Default: false (disabled)
  //
  // Dynamically changeable through SetOptions() API
  bool memtable_whole_key_filtering = false;
@ -421,7 +421,7 @@ struct AdvancedColumnFamilyOptions {
  // example would be updating the same key over and over again, in which case
  // the prefix can be the key itself.
  //
-  // Default: nullptr (disable)
+  // Default: nullptr (disabled)
  std::shared_ptr<const SliceTransform>
      memtable_insert_with_hint_prefix_extractor = nullptr;

--- a/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java
+++ b/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java
@ -569,6 +569,7 @@ public class DbBenchmark {
    options.setMinWriteBufferNumberToMerge(
        (Integer)flags_.get(Flag.min_write_buffer_number_to_merge));
    options.setMemtablePrefixBloomSizeRatio((Double) flags_.get(Flag.memtable_bloom_size_ratio));
+    options.setMemtableWholeKeyFiltering((Boolean) flags_.get(Flag.memtable_whole_key_filtering));
    options.setNumLevels(
        (Integer)flags_.get(Flag.num_levels));
    options.setTargetFileSizeBase(
@ -1190,6 +1191,12 @@ public class DbBenchmark {
        return Double.parseDouble(value);
      }
    },
+    memtable_whole_key_filtering(false, "Enable whole key bloom filter in memtable.") {
+      @Override
+      public Object parseValue(String value) {
+        return parseBoolean(value);
+      }
+    },
    cache_numshardbits(-1,"Number of shards for the block cache\n" +
        "\tis 2 ** cache_numshardbits. Negative means use default settings.\n" +
        "\tThis is applied only if FLAGS_cache_size is non-negative.") {
--- a/java/rocksjni/options.cc
+++ b/java/rocksjni/options.cc
@ -3124,6 +3124,29 @@ void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio(
      static_cast<double>(jmemtable_prefix_bloom_size_ratio);
 }

+/*
+ * Class:     org_rocksdb_Options
+ * Method:    memtableWholeKeyFiltering
+ * Signature: (J)Z
+ */
+jboolean Java_org_rocksdb_Options_memtableWholeKeyFiltering(JNIEnv*, jobject,
+                                                            jlong jhandle) {
+  return reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle)
+      ->memtable_whole_key_filtering;
+}
+
+/*
+ * Class:     org_rocksdb_Options
+ * Method:    setMemtableWholeKeyFiltering
+ * Signature: (JZ)V
+ */
+void Java_org_rocksdb_Options_setMemtableWholeKeyFiltering(
+    JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) {
+  reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle)
+      ->memtable_whole_key_filtering =
+      static_cast<bool>(jmemtable_whole_key_filtering);
+}
+
 /*
 * Class:     org_rocksdb_Options
 * Method:    bloomLocality
@ -4909,6 +4932,29 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio(
      static_cast<double>(jmemtable_prefix_bloom_size_ratio);
 }

+/*
+ * Class:     org_rocksdb_ColumnFamilyOptions
+ * Method:    memtableWholeKeyFiltering
+ * Signature: (J)Z
+ */
+jboolean Java_org_rocksdb_ColumnFamilyOptions_memtableWholeKeyFiltering(
+    JNIEnv*, jobject, jlong jhandle) {
+  return reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle)
+      ->memtable_whole_key_filtering;
+}
+
+/*
+ * Class:     org_rocksdb_ColumnFamilyOptions
+ * Method:    setMemtableWholeKeyFiltering
+ * Signature: (JZ)V
+ */
+void Java_org_rocksdb_ColumnFamilyOptions_setMemtableWholeKeyFiltering(
+    JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) {
+  reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle)
+      ->memtable_whole_key_filtering =
+      static_cast<bool>(jmemtable_whole_key_filtering);
+}
+
 /*
 * Class:     org_rocksdb_ColumnFamilyOptions
 * Method:    bloomLocality
--- a/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java
+++ b/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java
@ -60,9 +60,10 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
   * write_buffer_size * memtable_prefix_bloom_size_ratio.
   * If it is larger than 0.25, it is santinized to 0.25.
   *
-   * Default: 0 (disable)
+   * Default: 0 (disabled)
   *
-   * @param memtablePrefixBloomSizeRatio The ratio
+   * @param memtablePrefixBloomSizeRatio the ratio of memtable used by the
+   *     bloom filter, 0 means no bloom filter
   * @return the reference to the current options.
   */
  T setMemtablePrefixBloomSizeRatio(
@ -74,12 +75,32 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
   * write_buffer_size * memtable_prefix_bloom_size_ratio.
   * If it is larger than 0.25, it is santinized to 0.25.
   *
-   * Default: 0 (disable)
+   * Default: 0 (disabled)
   *
-   * @return the ratio
+   * @return the ratio of memtable used by the bloom filter
   */
  double memtablePrefixBloomSizeRatio();

+  /**
+   * Enable whole key bloom filter in memtable. Note this will only take effect
+   * if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering
+   * can potentially reduce CPU usage for point-look-ups.
+   *
+   * Default: false (disabled)
+   *
+   * @param memtableWholeKeyFiltering true if whole key bloom filter is enabled
+   *     in memtable
+   * @return the reference to the current options.
+   */
+  T setMemtableWholeKeyFiltering(boolean memtableWholeKeyFiltering);
+
+  /**
+   * Returns whether whole key bloom filter is enabled in memtable
+   *
+   * @return true if whole key bloom filter is enabled in memtable
+   */
+  boolean memtableWholeKeyFiltering();
+
  /**
   * Page size for huge page TLB for bloom in memtable. If &le; 0, not allocate
   * from huge page TLB but from malloc.
@ -546,7 +567,7 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
   * Dynamically changeable through
   * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}.
   *
-   * @return true iff blob files are enabled
+   * @return true if blob files are enabled
   */
  boolean enableBlobFiles();

@ -656,7 +677,7 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
   *
   * Default: false
   *
-   * @return true iff blob garbage collection is currently enabled.
+   * @return true if blob garbage collection is currently enabled.
   */
  boolean enableBlobGarbageCollection();

--- a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
+++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
@ -683,6 +683,17 @@ public class ColumnFamilyOptions extends RocksObject
    return memtablePrefixBloomSizeRatio(nativeHandle_);
  }

+  @Override
+  public ColumnFamilyOptions setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) {
+    setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering);
+    return this;
+  }
+
+  @Override
+  public boolean memtableWholeKeyFiltering() {
+    return memtableWholeKeyFiltering(nativeHandle_);
+  }
+
  @Override
  public ColumnFamilyOptions setBloomLocality(int bloomLocality) {
    setBloomLocality(nativeHandle_, bloomLocality);
@ -1319,6 +1330,8 @@ public class ColumnFamilyOptions extends RocksObject
  private native void setMemtablePrefixBloomSizeRatio(
      long handle, double memtablePrefixBloomSizeRatio);
  private native double memtablePrefixBloomSizeRatio(long handle);
+  private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering);
+  private native boolean memtableWholeKeyFiltering(long handle);
  private native void setBloomLocality(
      long handle, int bloomLocality);
  private native int bloomLocality(long handle);
--- a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
+++ b/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
@ -66,6 +66,7 @@ public class MutableColumnFamilyOptions
    write_buffer_size(ValueType.LONG),
    arena_block_size(ValueType.LONG),
    memtable_prefix_bloom_size_ratio(ValueType.DOUBLE),
+    memtable_whole_key_filtering(ValueType.BOOLEAN),
    @Deprecated memtable_prefix_bloom_bits(ValueType.INT),
    @Deprecated memtable_prefix_bloom_probes(ValueType.INT),
    memtable_huge_page_size(ValueType.LONG),
@ -226,6 +227,17 @@ public class MutableColumnFamilyOptions
      return getDouble(MemtableOption.memtable_prefix_bloom_size_ratio);
    }

+    @Override
+    public MutableColumnFamilyOptionsBuilder setMemtableWholeKeyFiltering(
+        final boolean memtableWholeKeyFiltering) {
+      return setBoolean(MemtableOption.memtable_whole_key_filtering, memtableWholeKeyFiltering);
+    }
+
+    @Override
+    public boolean memtableWholeKeyFiltering() {
+      return getBoolean(MemtableOption.memtable_whole_key_filtering);
+    }
+
    @Override
    public MutableColumnFamilyOptionsBuilder setMemtableHugePageSize(
        final long memtableHugePageSize) {
--- a/java/src/main/java/org/rocksdb/Options.java
+++ b/java/src/main/java/org/rocksdb/Options.java
@ -1638,6 +1638,17 @@ public class Options extends RocksObject
    return this;
  }

+  @Override
+  public boolean memtableWholeKeyFiltering() {
+    return memtableWholeKeyFiltering(nativeHandle_);
+  }
+
+  @Override
+  public Options setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) {
+    setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering);
+    return this;
+  }
+
  @Override
  public int bloomLocality() {
    return bloomLocality(nativeHandle_);
@ -2405,6 +2416,8 @@ public class Options extends RocksObject
  private native void setMemtablePrefixBloomSizeRatio(
      long handle, double memtablePrefixBloomSizeRatio);
  private native double memtablePrefixBloomSizeRatio(long handle);
+  private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering);
+  private native boolean memtableWholeKeyFiltering(long handle);
  private native void setBloomLocality(
      long handle, int bloomLocality);
  private native int bloomLocality(long handle);
--- a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
+++ b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
@ -331,6 +331,15 @@ public class ColumnFamilyOptionsTest {
    }
  }

+  @Test
+  public void memtableWholeKeyFiltering() {
+    try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
+      final boolean booleanValue = rand.nextBoolean();
+      opt.setMemtableWholeKeyFiltering(booleanValue);
+      assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue);
+    }
+  }
+
  @Test
  public void memtableHugePageSize() {
    try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
--- a/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java
+++ b/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java
@ -48,6 +48,7 @@ public class MutableOptionsGetSetTest {
                   .setBlobGarbageCollectionForceThreshold(0.80)
                   .setArenaBlockSize(42)
                   .setMemtablePrefixBloomSizeRatio(0.17)
+                   .setMemtableWholeKeyFiltering(false)
                   .setMemtableHugePageSize(3)
                   .setMaxSuccessiveMerges(4)
                   .setMaxWriteBufferNumber(12)
@ -70,6 +71,7 @@ public class MutableOptionsGetSetTest {
                   .setEnableBlobFiles(false)
                   .setArenaBlockSize(42)
                   .setMemtablePrefixBloomSizeRatio(0.236)
+                   .setMemtableWholeKeyFiltering(true)
                   .setMemtableHugePageSize(8)
                   .setMaxSuccessiveMerges(12)
                   .setMaxWriteBufferNumber(22)
@ -104,8 +106,9 @@ public class MutableOptionsGetSetTest {
        assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
        assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
        assertThat(builder1.arenaBlockSize()).isEqualTo(42);
-        assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
        assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);
+        assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false);
+        assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
        assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4);
        assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12);
        assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16);
@ -127,8 +130,9 @@ public class MutableOptionsGetSetTest {
        assertThat(builder2.enableBlobFiles()).isEqualTo(false);
        assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize);
        assertThat(builder2.arenaBlockSize()).isEqualTo(42);
-        assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
        assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236);
+        assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true);
+        assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
        assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12);
        assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22);
        assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160);
@ -192,6 +196,7 @@ public class MutableOptionsGetSetTest {
                .setBlobGarbageCollectionForceThreshold(0.80)
                .setArenaBlockSize(42)
                .setMemtablePrefixBloomSizeRatio(0.17)
+                .setMemtableWholeKeyFiltering(false)
                .setMemtableHugePageSize(3)
                .setMaxSuccessiveMerges(4)
                .setMaxWriteBufferNumber(12)
@ -215,8 +220,9 @@ public class MutableOptionsGetSetTest {
        assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
        assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
        assertThat(builder1.arenaBlockSize()).isEqualTo(42);
-        assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
        assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);
+        assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false);
+        assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
        assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4);
        assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12);
        assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16);
@ -237,6 +243,7 @@ public class MutableOptionsGetSetTest {
                .setEnableBlobFiles(false)
                .setArenaBlockSize(42)
                .setMemtablePrefixBloomSizeRatio(0.236)
+                .setMemtableWholeKeyFiltering(true)
                .setMemtableHugePageSize(8)
                .setMaxSuccessiveMerges(12)
                .setMaxWriteBufferNumber(22)
@ -258,8 +265,9 @@ public class MutableOptionsGetSetTest {
        assertThat(builder2.enableBlobFiles()).isEqualTo(false);
        assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize);
        assertThat(builder2.arenaBlockSize()).isEqualTo(42);
-        assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
        assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236);
+        assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true);
+        assertThat(builder2.memtableHugePageSize()).isEqualTo(8);
        assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12);
        assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22);
        assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160);
@ -306,6 +314,7 @@ public class MutableOptionsGetSetTest {
              .setBlobGarbageCollectionForceThreshold(0.80)
              .setArenaBlockSize(42)
              .setMemtablePrefixBloomSizeRatio(0.17)
+              .setMemtableWholeKeyFiltering(false)
              .setMemtableHugePageSize(3)
              .setMaxSuccessiveMerges(4)
              .setMaxWriteBufferNumber(12)
@ -328,8 +337,9 @@ public class MutableOptionsGetSetTest {
      assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
      assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
      assertThat(builder1.arenaBlockSize()).isEqualTo(42);
-      assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
      assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);
+      assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false);
+      assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
      assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4);
      assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12);
      assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16);
--- a/java/src/test/java/org/rocksdb/OptionsTest.java
+++ b/java/src/test/java/org/rocksdb/OptionsTest.java
@ -274,6 +274,15 @@ public class OptionsTest {
    }
  }

+  @Test
+  public void memtableWholeKeyFiltering() {
+    try (final Options opt = new Options()) {
+      final boolean booleanValue = rand.nextBoolean();
+      opt.setMemtableWholeKeyFiltering(booleanValue);
+      assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue);
+    }
+  }
+
  @Test
  public void memtableHugePageSize() {
    try (final Options opt = new Options()) {