From ac251aa641da13f408f4ace41f5130853b1b4e7a Mon Sep 17 00:00:00 2001 From: Levi Tamasi Date: Mon, 14 Feb 2022 09:11:59 -0800 Subject: [PATCH] Add Java bindings for blob compaction readahead size (#9554) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/9554 Test Plan: Added new unit tests. Reviewed By: mrambacher Differential Revision: D34197121 Pulled By: ltamasi fbshipit-source-id: 15056e26d632057a7c052a5024a560ba0eac554c --- java/rocksjni/options.cc | 58 +++++++++++++++++-- ...edMutableColumnFamilyOptionsInterface.java | 21 +++++++ .../java/org/rocksdb/ColumnFamilyOptions.java | 32 ++++++++++ .../rocksdb/MutableColumnFamilyOptions.java | 14 ++++- java/src/main/java/org/rocksdb/Options.java | 14 +++++ .../java/org/rocksdb/BlobOptionsTest.java | 32 ++++++---- .../MutableColumnFamilyOptionsTest.java | 3 +- .../org/rocksdb/MutableOptionsGetSetTest.java | 6 ++ 8 files changed, 163 insertions(+), 17 deletions(-) diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index 08b8817f9..42cb1e244 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -3700,7 +3700,7 @@ jlong Java_org_rocksdb_Options_minBlobSize(JNIEnv*, jobject, jlong jhandle) { /* * Class: org_rocksdb_Options - * Method: setMinBlobSize + * Method: setBlobFileSize * Signature: (JJ)V */ void Java_org_rocksdb_Options_setBlobFileSize(JNIEnv*, jobject, jlong jhandle, @@ -3711,7 +3711,7 @@ void Java_org_rocksdb_Options_setBlobFileSize(JNIEnv*, jobject, jlong jhandle, /* * Class: org_rocksdb_Options - * Method: minBlobSize + * Method: blobFileSize * Signature: (J)J */ jlong Java_org_rocksdb_Options_blobFileSize(JNIEnv*, jobject, jlong jhandle) { @@ -3816,6 +3816,29 @@ jdouble Java_org_rocksdb_Options_blobGarbageCollectionForceThreshold( return static_cast(opts->blob_garbage_collection_force_threshold); } +/* + * Class: org_rocksdb_Options + * Method: setBlobCompactionReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_Options_setBlobCompactionReadaheadSize( + JNIEnv*, jobject, jlong jhandle, jlong jblob_compaction_readahead_size) { + auto* opts = reinterpret_cast(jhandle); + opts->blob_compaction_readahead_size = + static_cast(jblob_compaction_readahead_size); +} + +/* + * Class: org_rocksdb_Options + * Method: blobCompactionReadaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_Options_blobCompactionReadaheadSize(JNIEnv*, jobject, + jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->blob_compaction_readahead_size); +} + ////////////////////////////////////////////////////////////////////////////// // ROCKSDB_NAMESPACE::ColumnFamilyOptions @@ -5447,7 +5470,7 @@ jlong Java_org_rocksdb_ColumnFamilyOptions_minBlobSize(JNIEnv*, jobject, /* * Class: org_rocksdb_ColumnFamilyOptions - * Method: setMinBlobSize + * Method: setBlobFileSize * Signature: (JJ)V */ void Java_org_rocksdb_ColumnFamilyOptions_setBlobFileSize( @@ -5459,7 +5482,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBlobFileSize( /* * Class: org_rocksdb_ColumnFamilyOptions - * Method: minBlobSize + * Method: blobFileSize * Signature: (J)J */ jlong Java_org_rocksdb_ColumnFamilyOptions_blobFileSize(JNIEnv*, jobject, @@ -5563,7 +5586,7 @@ void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionForceThreshold /* * Class: org_rocksdb_ColumnFamilyOptions - * Method: blobGarbageCollectionAgeCutoff + * Method: blobGarbageCollectionForceThreshold * Signature: (J)D */ jdouble @@ -5574,6 +5597,31 @@ Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionForceThreshold( return static_cast(opts->blob_garbage_collection_force_threshold); } +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBlobCompactionReadaheadSize + * Signature: (JJ)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBlobCompactionReadaheadSize( + JNIEnv*, jobject, jlong jhandle, jlong jblob_compaction_readahead_size) { + auto* opts = + reinterpret_cast(jhandle); + opts->blob_compaction_readahead_size = + static_cast(jblob_compaction_readahead_size); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: blobCompactionReadaheadSize + * Signature: (J)J + */ +jlong Java_org_rocksdb_ColumnFamilyOptions_blobCompactionReadaheadSize( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->blob_compaction_readahead_size); +} + ///////////////////////////////////////////////////////////////////// // ROCKSDB_NAMESPACE::DBOptions diff --git a/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java b/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java index d77aeef69..f871f65e0 100644 --- a/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java @@ -733,6 +733,27 @@ public interface AdvancedMutableColumnFamilyOptionsInterface< */ double blobGarbageCollectionForceThreshold(); + /** + * Set compaction readahead for blob files. + * + * Default: 0 + * + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param blobCompactionReadaheadSize the compaction readahead for blob files + * + * @return the reference to the current options. + */ + T setBlobCompactionReadaheadSize(final long blobCompactionReadaheadSize); + + /** + * Get compaction readahead for blob files. + * + * @return the current compaction readahead for blob files + */ + long blobCompactionReadaheadSize(); + // // END options for blobs (integrated BlobDB) // diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java index 726e7ca90..834239557 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java @@ -1209,6 +1209,35 @@ public class ColumnFamilyOptions extends RocksObject return blobGarbageCollectionForceThreshold(nativeHandle_); } + /** + * Set compaction readahead for blob files. + * + * Default: 0 + * + * Dynamically changeable through + * {@link RocksDB#setOptions(ColumnFamilyHandle, MutableColumnFamilyOptions)}. + * + * @param blobCompactionReadaheadSize the compaction readahead for blob files + * + * @return the reference to the current options. + */ + @Override + public ColumnFamilyOptions setBlobCompactionReadaheadSize( + final long blobCompactionReadaheadSize) { + setBlobCompactionReadaheadSize(nativeHandle_, blobCompactionReadaheadSize); + return this; + } + + /** + * Get compaction readahead for blob files. + * + * @return the current compaction readahead for blob files + */ + @Override + public long blobCompactionReadaheadSize() { + return blobCompactionReadaheadSize(nativeHandle_); + } + // // END options for blobs (integrated BlobDB) // @@ -1408,6 +1437,9 @@ public class ColumnFamilyOptions extends RocksObject private native void setBlobGarbageCollectionForceThreshold( final long nativeHandle_, final double blobGarbageCollectionForceThreshold); private native double blobGarbageCollectionForceThreshold(final long nativeHandle_); + private native void setBlobCompactionReadaheadSize( + final long nativeHandle_, final long blobCompactionReadaheadSize); + private native long blobCompactionReadaheadSize(final long nativeHandle_); // instance variables // NOTE: If you add new member variables, please update the copy constructor above! diff --git a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java index 4d422615e..a5019f1d0 100644 --- a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java @@ -120,7 +120,8 @@ public class MutableColumnFamilyOptions blob_compression_type(ValueType.ENUM), enable_blob_garbage_collection(ValueType.BOOLEAN), blob_garbage_collection_age_cutoff(ValueType.DOUBLE), - blob_garbage_collection_force_threshold(ValueType.DOUBLE); + blob_garbage_collection_force_threshold(ValueType.DOUBLE), + blob_compaction_readahead_size(ValueType.LONG); private final ValueType valueType; BlobOption(final ValueType valueType) { @@ -570,5 +571,16 @@ public class MutableColumnFamilyOptions public double blobGarbageCollectionForceThreshold() { return getDouble(BlobOption.blob_garbage_collection_force_threshold); } + + @Override + public MutableColumnFamilyOptionsBuilder setBlobCompactionReadaheadSize( + final long blobCompactionReadaheadSize) { + return setLong(BlobOption.blob_compaction_readahead_size, blobCompactionReadaheadSize); + } + + @Override + public long blobCompactionReadaheadSize() { + return getLong(BlobOption.blob_compaction_readahead_size); + } } } diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/rocksdb/Options.java index 1fab6d383..fcff02589 100644 --- a/java/src/main/java/org/rocksdb/Options.java +++ b/java/src/main/java/org/rocksdb/Options.java @@ -2071,6 +2071,17 @@ public class Options extends RocksObject return blobGarbageCollectionForceThreshold(nativeHandle_); } + @Override + public Options setBlobCompactionReadaheadSize(final long blobCompactionReadaheadSize) { + setBlobCompactionReadaheadSize(nativeHandle_, blobCompactionReadaheadSize); + return this; + } + + @Override + public long blobCompactionReadaheadSize() { + return blobCompactionReadaheadSize(nativeHandle_); + } + // // END options for blobs (integrated BlobDB) // @@ -2499,6 +2510,9 @@ public class Options extends RocksObject private native void setBlobGarbageCollectionForceThreshold( final long nativeHandle_, final double blobGarbageCollectionForceThreshold); private native double blobGarbageCollectionForceThreshold(final long nativeHandle_); + private native void setBlobCompactionReadaheadSize( + final long nativeHandle_, final long blobCompactionReadaheadSize); + private native long blobCompactionReadaheadSize(final long nativeHandle_); // instance variables // NOTE: If you add new member variables, please update the copy constructor above! diff --git a/java/src/test/java/org/rocksdb/BlobOptionsTest.java b/java/src/test/java/org/rocksdb/BlobOptionsTest.java index 371709ef3..1b23461ea 100644 --- a/java/src/test/java/org/rocksdb/BlobOptionsTest.java +++ b/java/src/test/java/org/rocksdb/BlobOptionsTest.java @@ -78,6 +78,7 @@ public class BlobOptionsTest { assertThat(options.blobFileSize()).isEqualTo(268435456L); assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(1.0); + assertThat(options.blobCompactionReadaheadSize()).isEqualTo(0); assertThat(options.setEnableBlobFiles(true)).isEqualTo(options); assertThat(options.setMinBlobSize(132768L)).isEqualTo(options); @@ -87,6 +88,7 @@ public class BlobOptionsTest { assertThat(options.setBlobFileSize(132768L)).isEqualTo(options); assertThat(options.setBlobGarbageCollectionAgeCutoff(0.89)).isEqualTo(options); assertThat(options.setBlobGarbageCollectionForceThreshold(0.80)).isEqualTo(options); + assertThat(options.setBlobCompactionReadaheadSize(262144L)).isEqualTo(options); assertThat(options.enableBlobFiles()).isEqualTo(true); assertThat(options.minBlobSize()).isEqualTo(132768L); @@ -95,6 +97,7 @@ public class BlobOptionsTest { assertThat(options.blobFileSize()).isEqualTo(132768L); assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(options.blobCompactionReadaheadSize()).isEqualTo(262144L); } } @@ -109,6 +112,7 @@ public class BlobOptionsTest { assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(268435456L); assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(1.0); + assertThat(columnFamilyOptions.blobCompactionReadaheadSize()).isEqualTo(0); assertThat(columnFamilyOptions.setEnableBlobFiles(true)).isEqualTo(columnFamilyOptions); assertThat(columnFamilyOptions.setMinBlobSize(132768L)).isEqualTo(columnFamilyOptions); @@ -121,6 +125,8 @@ public class BlobOptionsTest { .isEqualTo(columnFamilyOptions); assertThat(columnFamilyOptions.setBlobGarbageCollectionForceThreshold(0.80)) .isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setBlobCompactionReadaheadSize(262144L)) + .isEqualTo(columnFamilyOptions); assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(true); assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(132768L); @@ -130,6 +136,7 @@ public class BlobOptionsTest { assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(132768L); assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(columnFamilyOptions.blobCompactionReadaheadSize()).isEqualTo(262144L); } } @@ -139,44 +146,49 @@ public class BlobOptionsTest { MutableColumnFamilyOptions.builder(); builder.setEnableBlobFiles(true) .setMinBlobSize(1024) + .setBlobFileSize(132768) .setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION) .setEnableBlobGarbageCollection(true) .setBlobGarbageCollectionAgeCutoff(0.89) .setBlobGarbageCollectionForceThreshold(0.80) - .setBlobFileSize(132768); + .setBlobCompactionReadaheadSize(262144); assertThat(builder.enableBlobFiles()).isEqualTo(true); assertThat(builder.minBlobSize()).isEqualTo(1024); + assertThat(builder.blobFileSize()).isEqualTo(132768); assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION); assertThat(builder.enableBlobGarbageCollection()).isEqualTo(true); assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); - assertThat(builder.blobFileSize()).isEqualTo(132768); + assertThat(builder.blobCompactionReadaheadSize()).isEqualTo(262144); builder.setEnableBlobFiles(false) .setMinBlobSize(4096) + .setBlobFileSize(2048) .setBlobCompressionType(CompressionType.LZ4_COMPRESSION) .setEnableBlobGarbageCollection(false) .setBlobGarbageCollectionAgeCutoff(0.91) .setBlobGarbageCollectionForceThreshold(0.96) - .setBlobFileSize(2048); + .setBlobCompactionReadaheadSize(1024); assertThat(builder.enableBlobFiles()).isEqualTo(false); assertThat(builder.minBlobSize()).isEqualTo(4096); + assertThat(builder.blobFileSize()).isEqualTo(2048); assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.LZ4_COMPRESSION); assertThat(builder.enableBlobGarbageCollection()).isEqualTo(false); assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.91); assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.96); - assertThat(builder.blobFileSize()).isEqualTo(2048); + assertThat(builder.blobCompactionReadaheadSize()).isEqualTo(1024); final MutableColumnFamilyOptions options = builder.build(); assertThat(options.getKeys()) - .isEqualTo(new String[] {"enable_blob_files", "min_blob_size", "blob_compression_type", - "enable_blob_garbage_collection", "blob_garbage_collection_age_cutoff", - "blob_garbage_collection_force_threshold", "blob_file_size"}); + .isEqualTo(new String[] {"enable_blob_files", "min_blob_size", "blob_file_size", + "blob_compression_type", "enable_blob_garbage_collection", + "blob_garbage_collection_age_cutoff", "blob_garbage_collection_force_threshold", + "blob_compaction_readahead_size"}); assertThat(options.getValues()) - .isEqualTo( - new String[] {"false", "4096", "LZ4_COMPRESSION", "false", "0.91", "0.96", "2048"}); + .isEqualTo(new String[] { + "false", "4096", "2048", "LZ4_COMPRESSION", "false", "0.91", "0.96", "1024"}); } /** @@ -310,4 +322,4 @@ public class BlobOptionsTest { } } } -} \ No newline at end of file +} diff --git a/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java b/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java index 9cbec5dd1..3109d48fd 100644 --- a/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java +++ b/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java @@ -97,7 +97,7 @@ public class MutableColumnFamilyOptionsTest { final String optionsString = "bottommost_compression=kDisableCompressionOption; sample_for_compression=0; " + "blob_garbage_collection_age_cutoff=0.250000; blob_garbage_collection_force_threshold=0.800000; arena_block_size=1048576; enable_blob_garbage_collection=false; " - + "level0_stop_writes_trigger=36; min_blob_size=65536; " + + "level0_stop_writes_trigger=36; min_blob_size=65536; blob_compaction_readahead_size=262144; " + "compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;" + "compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;}; " + "target_file_size_base=67108864; max_bytes_for_level_base=268435456; memtable_whole_key_filtering=false; " @@ -131,6 +131,7 @@ public class MutableColumnFamilyOptionsTest { assertThat(cf.enableBlobGarbageCollection()).isEqualTo(false); assertThat(cf.level0StopWritesTrigger()).isEqualTo(36); assertThat(cf.minBlobSize()).isEqualTo(65536); + assertThat(cf.blobCompactionReadaheadSize()).isEqualTo(262144); assertThat(cf.targetFileSizeBase()).isEqualTo(67108864); assertThat(cf.maxBytesForLevelBase()).isEqualTo(268435456); assertThat(cf.softPendingCompactionBytesLimit()).isEqualTo(68719476736L); diff --git a/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java b/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java index ce8211c1e..0d6dbafee 100644 --- a/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java +++ b/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java @@ -46,6 +46,7 @@ public class MutableOptionsGetSetTest { .setEnableBlobFiles(true) .setBlobGarbageCollectionAgeCutoff(0.25) .setBlobGarbageCollectionForceThreshold(0.80) + .setBlobCompactionReadaheadSize(262144) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.17) .setMemtableWholeKeyFiltering(false) @@ -104,6 +105,7 @@ public class MutableOptionsGetSetTest { assertThat(builder1.enableBlobFiles()).isEqualTo(true); assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(builder1.blobCompactionReadaheadSize()).isEqualTo(262144); assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder1.arenaBlockSize()).isEqualTo(42); assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); @@ -194,6 +196,7 @@ public class MutableOptionsGetSetTest { .setEnableBlobFiles(true) .setBlobGarbageCollectionAgeCutoff(0.25) .setBlobGarbageCollectionForceThreshold(0.80) + .setBlobCompactionReadaheadSize(262144) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.17) .setMemtableWholeKeyFiltering(false) @@ -218,6 +221,7 @@ public class MutableOptionsGetSetTest { assertThat(builder1.enableBlobFiles()).isEqualTo(true); assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(builder1.blobCompactionReadaheadSize()).isEqualTo(262144); assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder1.arenaBlockSize()).isEqualTo(42); assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); @@ -312,6 +316,7 @@ public class MutableOptionsGetSetTest { .setEnableBlobFiles(true) .setBlobGarbageCollectionAgeCutoff(0.25) .setBlobGarbageCollectionForceThreshold(0.80) + .setBlobCompactionReadaheadSize(131072) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.17) .setMemtableWholeKeyFiltering(false) @@ -335,6 +340,7 @@ public class MutableOptionsGetSetTest { assertThat(builder1.enableBlobFiles()).isEqualTo(true); assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); + assertThat(builder1.blobCompactionReadaheadSize()).isEqualTo(131072); assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder1.arenaBlockSize()).isEqualTo(42); assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17);