From 83ff350ff2f0cc4945a3a1f5c8f246806f4b3758 Mon Sep 17 00:00:00 2001 From: Jermy Li Date: Fri, 4 Feb 2022 16:00:19 -0800 Subject: [PATCH] jni: expose memtable_whole_key_filtering option (#9394) Summary: refer to: https://github.com/facebook/rocksdb/wiki/Prefix-Seek#configure-prefix-bloom-filter Pull Request resolved: https://github.com/facebook/rocksdb/pull/9394 Reviewed By: mrambacher Differential Revision: D33671533 Pulled By: ajkr fbshipit-source-id: d90db1712efdd5dd65020329867381d6b3cf2626 --- include/rocksdb/advanced_options.h | 6 +-- .../org/rocksdb/benchmark/DbBenchmark.java | 7 +++ java/rocksjni/options.cc | 46 +++++++++++++++++++ ...edMutableColumnFamilyOptionsInterface.java | 33 ++++++++++--- .../java/org/rocksdb/ColumnFamilyOptions.java | 13 ++++++ .../rocksdb/MutableColumnFamilyOptions.java | 12 +++++ java/src/main/java/org/rocksdb/Options.java | 13 ++++++ .../org/rocksdb/ColumnFamilyOptionsTest.java | 9 ++++ .../org/rocksdb/MutableOptionsGetSetTest.java | 20 ++++++-- .../test/java/org/rocksdb/OptionsTest.java | 9 ++++ 10 files changed, 154 insertions(+), 14 deletions(-) diff --git a/include/rocksdb/advanced_options.h b/include/rocksdb/advanced_options.h index ccffb026d..fd080b6f1 100644 --- a/include/rocksdb/advanced_options.h +++ b/include/rocksdb/advanced_options.h @@ -377,7 +377,7 @@ struct AdvancedColumnFamilyOptions { // // If this value is larger than 0.25, it is sanitized to 0.25. // - // Default: 0 (disable) + // Default: 0 (disabled) // // Dynamically changeable through SetOptions() API double memtable_prefix_bloom_size_ratio = 0.0; @@ -386,7 +386,7 @@ struct AdvancedColumnFamilyOptions { // if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering // can potentially reduce CPU usage for point-look-ups. // - // Default: false (disable) + // Default: false (disabled) // // Dynamically changeable through SetOptions() API bool memtable_whole_key_filtering = false; @@ -421,7 +421,7 @@ struct AdvancedColumnFamilyOptions { // example would be updating the same key over and over again, in which case // the prefix can be the key itself. // - // Default: nullptr (disable) + // Default: nullptr (disabled) std::shared_ptr memtable_insert_with_hint_prefix_extractor = nullptr; diff --git a/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java b/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java index 04119c5f4..070f0fe75 100644 --- a/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java +++ b/java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java @@ -569,6 +569,7 @@ public class DbBenchmark { options.setMinWriteBufferNumberToMerge( (Integer)flags_.get(Flag.min_write_buffer_number_to_merge)); options.setMemtablePrefixBloomSizeRatio((Double) flags_.get(Flag.memtable_bloom_size_ratio)); + options.setMemtableWholeKeyFiltering((Boolean) flags_.get(Flag.memtable_whole_key_filtering)); options.setNumLevels( (Integer)flags_.get(Flag.num_levels)); options.setTargetFileSizeBase( @@ -1190,6 +1191,12 @@ public class DbBenchmark { return Double.parseDouble(value); } }, + memtable_whole_key_filtering(false, "Enable whole key bloom filter in memtable.") { + @Override + public Object parseValue(String value) { + return parseBoolean(value); + } + }, cache_numshardbits(-1,"Number of shards for the block cache\n" + "\tis 2 ** cache_numshardbits. Negative means use default settings.\n" + "\tThis is applied only if FLAGS_cache_size is non-negative.") { diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index dd51ea6db..bb7c80ee3 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -3124,6 +3124,29 @@ void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio( static_cast(jmemtable_prefix_bloom_size_ratio); } +/* + * Class: org_rocksdb_Options + * Method: memtableWholeKeyFiltering + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_Options_memtableWholeKeyFiltering(JNIEnv*, jobject, + jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_whole_key_filtering; +} + +/* + * Class: org_rocksdb_Options + * Method: setMemtableWholeKeyFiltering + * Signature: (JZ)V + */ +void Java_org_rocksdb_Options_setMemtableWholeKeyFiltering( + JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) { + reinterpret_cast(jhandle) + ->memtable_whole_key_filtering = + static_cast(jmemtable_whole_key_filtering); +} + /* * Class: org_rocksdb_Options * Method: bloomLocality @@ -4909,6 +4932,29 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio( static_cast(jmemtable_prefix_bloom_size_ratio); } +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: memtableWholeKeyFiltering + * Signature: (J)Z + */ +jboolean Java_org_rocksdb_ColumnFamilyOptions_memtableWholeKeyFiltering( + JNIEnv*, jobject, jlong jhandle) { + return reinterpret_cast(jhandle) + ->memtable_whole_key_filtering; +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setMemtableWholeKeyFiltering + * Signature: (JZ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setMemtableWholeKeyFiltering( + JNIEnv*, jobject, jlong jhandle, jboolean jmemtable_whole_key_filtering) { + reinterpret_cast(jhandle) + ->memtable_whole_key_filtering = + static_cast(jmemtable_whole_key_filtering); +} + /* * Class: org_rocksdb_ColumnFamilyOptions * Method: bloomLocality diff --git a/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java b/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java index afd1e37de..d77aeef69 100644 --- a/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java @@ -60,9 +60,10 @@ public interface AdvancedMutableColumnFamilyOptionsInterface< * write_buffer_size * memtable_prefix_bloom_size_ratio. * If it is larger than 0.25, it is santinized to 0.25. * - * Default: 0 (disable) + * Default: 0 (disabled) * - * @param memtablePrefixBloomSizeRatio The ratio + * @param memtablePrefixBloomSizeRatio the ratio of memtable used by the + * bloom filter, 0 means no bloom filter * @return the reference to the current options. */ T setMemtablePrefixBloomSizeRatio( @@ -74,12 +75,32 @@ public interface AdvancedMutableColumnFamilyOptionsInterface< * write_buffer_size * memtable_prefix_bloom_size_ratio. * If it is larger than 0.25, it is santinized to 0.25. * - * Default: 0 (disable) + * Default: 0 (disabled) * - * @return the ratio + * @return the ratio of memtable used by the bloom filter */ double memtablePrefixBloomSizeRatio(); + /** + * Enable whole key bloom filter in memtable. Note this will only take effect + * if memtable_prefix_bloom_size_ratio is not 0. Enabling whole key filtering + * can potentially reduce CPU usage for point-look-ups. + * + * Default: false (disabled) + * + * @param memtableWholeKeyFiltering true if whole key bloom filter is enabled + * in memtable + * @return the reference to the current options. + */ + T setMemtableWholeKeyFiltering(boolean memtableWholeKeyFiltering); + + /** + * Returns whether whole key bloom filter is enabled in memtable + * + * @return true if whole key bloom filter is enabled in memtable + */ + boolean memtableWholeKeyFiltering(); + /** * Page size for huge page TLB for bloom in memtable. If ≤ 0, not allocate * from huge page TLB but from malloc. @@ -546,7 +567,7 @@ public interface AdvancedMutableColumnFamilyOptionsInterface< * Dynamically changeable through * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. * - * @return true iff blob files are enabled + * @return true if blob files are enabled */ boolean enableBlobFiles(); @@ -656,7 +677,7 @@ public interface AdvancedMutableColumnFamilyOptionsInterface< * * Default: false * - * @return true iff blob garbage collection is currently enabled. + * @return true if blob garbage collection is currently enabled. */ boolean enableBlobGarbageCollection(); diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java index fa807ae67..a77662366 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java @@ -683,6 +683,17 @@ public class ColumnFamilyOptions extends RocksObject return memtablePrefixBloomSizeRatio(nativeHandle_); } + @Override + public ColumnFamilyOptions setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) { + setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering); + return this; + } + + @Override + public boolean memtableWholeKeyFiltering() { + return memtableWholeKeyFiltering(nativeHandle_); + } + @Override public ColumnFamilyOptions setBloomLocality(int bloomLocality) { setBloomLocality(nativeHandle_, bloomLocality); @@ -1319,6 +1330,8 @@ public class ColumnFamilyOptions extends RocksObject private native void setMemtablePrefixBloomSizeRatio( long handle, double memtablePrefixBloomSizeRatio); private native double memtablePrefixBloomSizeRatio(long handle); + private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering); + private native boolean memtableWholeKeyFiltering(long handle); private native void setBloomLocality( long handle, int bloomLocality); private native int bloomLocality(long handle); diff --git a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java index eaa1c4fd8..4d422615e 100644 --- a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java @@ -66,6 +66,7 @@ public class MutableColumnFamilyOptions write_buffer_size(ValueType.LONG), arena_block_size(ValueType.LONG), memtable_prefix_bloom_size_ratio(ValueType.DOUBLE), + memtable_whole_key_filtering(ValueType.BOOLEAN), @Deprecated memtable_prefix_bloom_bits(ValueType.INT), @Deprecated memtable_prefix_bloom_probes(ValueType.INT), memtable_huge_page_size(ValueType.LONG), @@ -226,6 +227,17 @@ public class MutableColumnFamilyOptions return getDouble(MemtableOption.memtable_prefix_bloom_size_ratio); } + @Override + public MutableColumnFamilyOptionsBuilder setMemtableWholeKeyFiltering( + final boolean memtableWholeKeyFiltering) { + return setBoolean(MemtableOption.memtable_whole_key_filtering, memtableWholeKeyFiltering); + } + + @Override + public boolean memtableWholeKeyFiltering() { + return getBoolean(MemtableOption.memtable_whole_key_filtering); + } + @Override public MutableColumnFamilyOptionsBuilder setMemtableHugePageSize( final long memtableHugePageSize) { diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/rocksdb/Options.java index 671d7525c..ffe779453 100644 --- a/java/src/main/java/org/rocksdb/Options.java +++ b/java/src/main/java/org/rocksdb/Options.java @@ -1638,6 +1638,17 @@ public class Options extends RocksObject return this; } + @Override + public boolean memtableWholeKeyFiltering() { + return memtableWholeKeyFiltering(nativeHandle_); + } + + @Override + public Options setMemtableWholeKeyFiltering(final boolean memtableWholeKeyFiltering) { + setMemtableWholeKeyFiltering(nativeHandle_, memtableWholeKeyFiltering); + return this; + } + @Override public int bloomLocality() { return bloomLocality(nativeHandle_); @@ -2405,6 +2416,8 @@ public class Options extends RocksObject private native void setMemtablePrefixBloomSizeRatio( long handle, double memtablePrefixBloomSizeRatio); private native double memtablePrefixBloomSizeRatio(long handle); + private native void setMemtableWholeKeyFiltering(long handle, boolean memtableWholeKeyFiltering); + private native boolean memtableWholeKeyFiltering(long handle); private native void setBloomLocality( long handle, int bloomLocality); private native int bloomLocality(long handle); diff --git a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java index 0919d7485..2ad669d33 100644 --- a/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java +++ b/java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java @@ -331,6 +331,15 @@ public class ColumnFamilyOptionsTest { } } + @Test + public void memtableWholeKeyFiltering() { + try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { + final boolean booleanValue = rand.nextBoolean(); + opt.setMemtableWholeKeyFiltering(booleanValue); + assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue); + } + } + @Test public void memtableHugePageSize() { try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { diff --git a/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java b/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java index 95b1209e5..ce8211c1e 100644 --- a/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java +++ b/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java @@ -48,6 +48,7 @@ public class MutableOptionsGetSetTest { .setBlobGarbageCollectionForceThreshold(0.80) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.17) + .setMemtableWholeKeyFiltering(false) .setMemtableHugePageSize(3) .setMaxSuccessiveMerges(4) .setMaxWriteBufferNumber(12) @@ -70,6 +71,7 @@ public class MutableOptionsGetSetTest { .setEnableBlobFiles(false) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.236) + .setMemtableWholeKeyFiltering(true) .setMemtableHugePageSize(8) .setMaxSuccessiveMerges(12) .setMaxWriteBufferNumber(22) @@ -104,8 +106,9 @@ public class MutableOptionsGetSetTest { assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder1.arenaBlockSize()).isEqualTo(42); - assertThat(builder1.memtableHugePageSize()).isEqualTo(3); assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); + assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false); + assertThat(builder1.memtableHugePageSize()).isEqualTo(3); assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4); assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12); assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16); @@ -127,8 +130,9 @@ public class MutableOptionsGetSetTest { assertThat(builder2.enableBlobFiles()).isEqualTo(false); assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder2.arenaBlockSize()).isEqualTo(42); - assertThat(builder2.memtableHugePageSize()).isEqualTo(8); assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236); + assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true); + assertThat(builder2.memtableHugePageSize()).isEqualTo(8); assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12); assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22); assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160); @@ -192,6 +196,7 @@ public class MutableOptionsGetSetTest { .setBlobGarbageCollectionForceThreshold(0.80) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.17) + .setMemtableWholeKeyFiltering(false) .setMemtableHugePageSize(3) .setMaxSuccessiveMerges(4) .setMaxWriteBufferNumber(12) @@ -215,8 +220,9 @@ public class MutableOptionsGetSetTest { assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder1.arenaBlockSize()).isEqualTo(42); - assertThat(builder1.memtableHugePageSize()).isEqualTo(3); assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); + assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false); + assertThat(builder1.memtableHugePageSize()).isEqualTo(3); assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4); assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12); assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16); @@ -237,6 +243,7 @@ public class MutableOptionsGetSetTest { .setEnableBlobFiles(false) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.236) + .setMemtableWholeKeyFiltering(true) .setMemtableHugePageSize(8) .setMaxSuccessiveMerges(12) .setMaxWriteBufferNumber(22) @@ -258,8 +265,9 @@ public class MutableOptionsGetSetTest { assertThat(builder2.enableBlobFiles()).isEqualTo(false); assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder2.arenaBlockSize()).isEqualTo(42); - assertThat(builder2.memtableHugePageSize()).isEqualTo(8); assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236); + assertThat(builder2.memtableWholeKeyFiltering()).isEqualTo(true); + assertThat(builder2.memtableHugePageSize()).isEqualTo(8); assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12); assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22); assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160); @@ -306,6 +314,7 @@ public class MutableOptionsGetSetTest { .setBlobGarbageCollectionForceThreshold(0.80) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.17) + .setMemtableWholeKeyFiltering(false) .setMemtableHugePageSize(3) .setMaxSuccessiveMerges(4) .setMaxWriteBufferNumber(12) @@ -328,8 +337,9 @@ public class MutableOptionsGetSetTest { assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder1.arenaBlockSize()).isEqualTo(42); - assertThat(builder1.memtableHugePageSize()).isEqualTo(3); assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); + assertThat(builder1.memtableWholeKeyFiltering()).isEqualTo(false); + assertThat(builder1.memtableHugePageSize()).isEqualTo(3); assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4); assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12); assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16); diff --git a/java/src/test/java/org/rocksdb/OptionsTest.java b/java/src/test/java/org/rocksdb/OptionsTest.java index 32c9aace0..c74d632cf 100644 --- a/java/src/test/java/org/rocksdb/OptionsTest.java +++ b/java/src/test/java/org/rocksdb/OptionsTest.java @@ -274,6 +274,15 @@ public class OptionsTest { } } + @Test + public void memtableWholeKeyFiltering() { + try (final Options opt = new Options()) { + final boolean booleanValue = rand.nextBoolean(); + opt.setMemtableWholeKeyFiltering(booleanValue); + assertThat(opt.memtableWholeKeyFiltering()).isEqualTo(booleanValue); + } + } + @Test public void memtableHugePageSize() { try (final Options opt = new Options()) {