From 73e6b89fad5ff2d9a0fdb06bdf2dad5b7d9b2a8b Mon Sep 17 00:00:00 2001 From: Alan Paxton Date: Mon, 1 Nov 2021 11:58:03 -0700 Subject: [PATCH] Java wrapper for blob_gc_force_threshold as blobGarbageCollectionForceThreshold (#9109) Summary: Extra option added as a supplement to https://github.com/facebook/rocksdb/pull/8999 Closes https://github.com/facebook/rocksdb/issues/8221 Pull Request resolved: https://github.com/facebook/rocksdb/pull/9109 Reviewed By: mrambacher Differential Revision: D32065039 Pulled By: ltamasi fbshipit-source-id: 6c484050a30fe0523850a8a3c95dc85b0a501362 --- java/rocksjni/options.cc | 51 +++++++++++++++++++ ...edMutableColumnFamilyOptionsInterface.java | 25 +++++++++ .../java/org/rocksdb/ColumnFamilyOptions.java | 36 +++++++++++++ .../rocksdb/MutableColumnFamilyOptions.java | 15 +++++- java/src/main/java/org/rocksdb/Options.java | 19 ++++++- .../java/org/rocksdb/BlobOptionsTest.java | 16 +++++- .../MutableColumnFamilyOptionsTest.java | 3 +- .../org/rocksdb/MutableOptionsGetSetTest.java | 12 +++++ 8 files changed, 171 insertions(+), 6 deletions(-) diff --git a/java/rocksjni/options.cc b/java/rocksjni/options.cc index 1040e788d..63fcda41f 100644 --- a/java/rocksjni/options.cc +++ b/java/rocksjni/options.cc @@ -3837,6 +3837,30 @@ jdouble Java_org_rocksdb_Options_blobGarbageCollectionAgeCutoff(JNIEnv*, return static_cast(opts->blob_garbage_collection_age_cutoff); } +/* + * Class: org_rocksdb_Options + * Method: setBlobGarbageCollectionForceThreshold + * Signature: (JD)V + */ +void Java_org_rocksdb_Options_setBlobGarbageCollectionForceThreshold( + JNIEnv*, jobject, jlong jhandle, + jdouble jblob_garbage_collection_force_threshold) { + auto* opts = reinterpret_cast(jhandle); + opts->blob_garbage_collection_force_threshold = + static_cast(jblob_garbage_collection_force_threshold); +} + +/* + * Class: org_rocksdb_Options + * Method: blobGarbageCollectionForceThreshold + * Signature: (J)D + */ +jdouble Java_org_rocksdb_Options_blobGarbageCollectionForceThreshold( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = reinterpret_cast(jhandle); + return static_cast(opts->blob_garbage_collection_force_threshold); +} + ////////////////////////////////////////////////////////////////////////////// // ROCKSDB_NAMESPACE::ColumnFamilyOptions @@ -5545,6 +5569,33 @@ jdouble Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionAgeCutoff( return static_cast(opts->blob_garbage_collection_age_cutoff); } +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: setBlobGarbageCollectionForceThreshold + * Signature: (JD)V + */ +void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionForceThreshold( + JNIEnv*, jobject, jlong jhandle, + jdouble jblob_garbage_collection_force_threshold) { + auto* opts = + reinterpret_cast(jhandle); + opts->blob_garbage_collection_force_threshold = + static_cast(jblob_garbage_collection_force_threshold); +} + +/* + * Class: org_rocksdb_ColumnFamilyOptions + * Method: blobGarbageCollectionAgeCutoff + * Signature: (J)D + */ +jdouble +Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionForceThreshold( + JNIEnv*, jobject, jlong jhandle) { + auto* opts = + reinterpret_cast(jhandle); + return static_cast(opts->blob_garbage_collection_force_threshold); +} + ///////////////////////////////////////////////////////////////////// // ROCKSDB_NAMESPACE::DBOptions diff --git a/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java b/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java index 7adbc2b71..afd1e37de 100644 --- a/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java +++ b/java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java @@ -687,6 +687,31 @@ public interface AdvancedMutableColumnFamilyOptionsInterface< */ double blobGarbageCollectionAgeCutoff(); + /** + * If the ratio of garbage in the oldest blob files exceeds this threshold, + * targeted compactions are scheduled in order to force garbage collecting + * the blob files in question, assuming they are all eligible based on the + * value of {@link #blobGarbageCollectionAgeCutoff} above. This option is + * currently only supported with leveled compactions. + * + * Note that {@link #enableBlobGarbageCollection} has to be set in order for this + * option to have any effect. + * + * Default: 1.0 + * + * Dynamically changeable through the SetOptions() API + * + * @param blobGarbageCollectionForceThreshold new value for the threshold + * @return the reference to the current options + */ + T setBlobGarbageCollectionForceThreshold(double blobGarbageCollectionForceThreshold); + + /** + * Get the current value for the {@link #blobGarbageCollectionForceThreshold} + * @return the current threshold at which garbage collection of blobs is forced + */ + double blobGarbageCollectionForceThreshold(); + // // END options for blobs (integrated BlobDB) // diff --git a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java index ba31bca36..9bb7e0942 100644 --- a/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/ColumnFamilyOptions.java @@ -1164,6 +1164,39 @@ public class ColumnFamilyOptions extends RocksObject return blobGarbageCollectionAgeCutoff(nativeHandle_); } + /** + * If the ratio of garbage in the oldest blob files exceeds this threshold, + * targeted compactions are scheduled in order to force garbage collecting + * the blob files in question, assuming they are all eligible based on the + * value of {@link #blobGarbageCollectionAgeCutoff} above. This option is + * currently only supported with leveled compactions. + * + * Note that {@link #enableBlobGarbageCollection} has to be set in order for this + * option to have any effect. + * + * Default: 1.0 + * + * Dynamically changeable through the SetOptions() API + * + * @param blobGarbageCollectionForceThreshold new value for the threshold + * @return the reference to the current options + */ + @Override + public ColumnFamilyOptions setBlobGarbageCollectionForceThreshold( + final double blobGarbageCollectionForceThreshold) { + setBlobGarbageCollectionForceThreshold(nativeHandle_, blobGarbageCollectionForceThreshold); + return this; + } + + /** + * Get the current value for the {@link #blobGarbageCollectionForceThreshold} + * @return the current threshold at which garbage collection of blobs is forced + */ + @Override + public double blobGarbageCollectionForceThreshold() { + return blobGarbageCollectionForceThreshold(nativeHandle_); + } + // // END options for blobs (integrated BlobDB) // @@ -1358,6 +1391,9 @@ public class ColumnFamilyOptions extends RocksObject private native void setBlobGarbageCollectionAgeCutoff( final long nativeHandle_, final double blobGarbageCollectionAgeCutoff); private native double blobGarbageCollectionAgeCutoff(final long nativeHandle_); + private native void setBlobGarbageCollectionForceThreshold( + final long nativeHandle_, final double blobGarbageCollectionForceThreshold); + private native double blobGarbageCollectionForceThreshold(final long nativeHandle_); // instance variables // NOTE: If you add new member variables, please update the copy constructor above! diff --git a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java b/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java index b0d078c70..d5d5ec156 100644 --- a/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java +++ b/java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java @@ -119,7 +119,8 @@ public class MutableColumnFamilyOptions blob_file_size(ValueType.LONG), blob_compression_type(ValueType.ENUM), enable_blob_garbage_collection(ValueType.BOOLEAN), - blob_garbage_collection_age_cutoff(ValueType.DOUBLE); + blob_garbage_collection_age_cutoff(ValueType.DOUBLE), + blob_garbage_collection_force_threshold(ValueType.DOUBLE); private final ValueType valueType; BlobOption(final ValueType valueType) { @@ -546,5 +547,17 @@ public class MutableColumnFamilyOptions public double blobGarbageCollectionAgeCutoff() { return getDouble(BlobOption.blob_garbage_collection_age_cutoff); } + + @Override + public MutableColumnFamilyOptionsBuilder setBlobGarbageCollectionForceThreshold( + final double blobGarbageCollectionForceThreshold) { + return setDouble( + BlobOption.blob_garbage_collection_force_threshold, blobGarbageCollectionForceThreshold); + } + + @Override + public double blobGarbageCollectionForceThreshold() { + return getDouble(BlobOption.blob_garbage_collection_force_threshold); + } } } diff --git a/java/src/main/java/org/rocksdb/Options.java b/java/src/main/java/org/rocksdb/Options.java index b12f053aa..f02697d77 100644 --- a/java/src/main/java/org/rocksdb/Options.java +++ b/java/src/main/java/org/rocksdb/Options.java @@ -2077,7 +2077,7 @@ public class Options extends RocksObject } @Override - public Options setBlobGarbageCollectionAgeCutoff(double blobGarbageCollectionAgeCutoff) { + public Options setBlobGarbageCollectionAgeCutoff(final double blobGarbageCollectionAgeCutoff) { setBlobGarbageCollectionAgeCutoff(nativeHandle_, blobGarbageCollectionAgeCutoff); return this; } @@ -2087,6 +2087,18 @@ public class Options extends RocksObject return blobGarbageCollectionAgeCutoff(nativeHandle_); } + @Override + public Options setBlobGarbageCollectionForceThreshold( + final double blobGarbageCollectionForceThreshold) { + setBlobGarbageCollectionForceThreshold(nativeHandle_, blobGarbageCollectionForceThreshold); + return this; + } + + @Override + public double blobGarbageCollectionForceThreshold() { + return blobGarbageCollectionForceThreshold(nativeHandle_); + } + // // END options for blobs (integrated BlobDB) // @@ -2517,8 +2529,11 @@ public class Options extends RocksObject final long nativeHandle_, final boolean enableBlobGarbageCollection); private native boolean enableBlobGarbageCollection(final long nativeHandle_); private native void setBlobGarbageCollectionAgeCutoff( - final long nativeHandle_, double blobGarbageCollectionAgeCutoff); + final long nativeHandle_, final double blobGarbageCollectionAgeCutoff); private native double blobGarbageCollectionAgeCutoff(final long nativeHandle_); + private native void setBlobGarbageCollectionForceThreshold( + final long nativeHandle_, final double blobGarbageCollectionForceThreshold); + private native double blobGarbageCollectionForceThreshold(final long nativeHandle_); // instance variables // NOTE: If you add new member variables, please update the copy constructor above! diff --git a/java/src/test/java/org/rocksdb/BlobOptionsTest.java b/java/src/test/java/org/rocksdb/BlobOptionsTest.java index e096b3f73..371709ef3 100644 --- a/java/src/test/java/org/rocksdb/BlobOptionsTest.java +++ b/java/src/test/java/org/rocksdb/BlobOptionsTest.java @@ -77,6 +77,7 @@ public class BlobOptionsTest { assertThat(options.enableBlobGarbageCollection()).isEqualTo(false); assertThat(options.blobFileSize()).isEqualTo(268435456L); assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(1.0); assertThat(options.setEnableBlobFiles(true)).isEqualTo(options); assertThat(options.setMinBlobSize(132768L)).isEqualTo(options); @@ -85,6 +86,7 @@ public class BlobOptionsTest { assertThat(options.setEnableBlobGarbageCollection(true)).isEqualTo(options); assertThat(options.setBlobFileSize(132768L)).isEqualTo(options); assertThat(options.setBlobGarbageCollectionAgeCutoff(0.89)).isEqualTo(options); + assertThat(options.setBlobGarbageCollectionForceThreshold(0.80)).isEqualTo(options); assertThat(options.enableBlobFiles()).isEqualTo(true); assertThat(options.minBlobSize()).isEqualTo(132768L); @@ -92,6 +94,7 @@ public class BlobOptionsTest { assertThat(options.enableBlobGarbageCollection()).isEqualTo(true); assertThat(options.blobFileSize()).isEqualTo(132768L); assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); + assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); } } @@ -105,6 +108,7 @@ public class BlobOptionsTest { assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(false); assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(268435456L); assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(1.0); assertThat(columnFamilyOptions.setEnableBlobFiles(true)).isEqualTo(columnFamilyOptions); assertThat(columnFamilyOptions.setMinBlobSize(132768L)).isEqualTo(columnFamilyOptions); @@ -115,6 +119,8 @@ public class BlobOptionsTest { assertThat(columnFamilyOptions.setBlobFileSize(132768L)).isEqualTo(columnFamilyOptions); assertThat(columnFamilyOptions.setBlobGarbageCollectionAgeCutoff(0.89)) .isEqualTo(columnFamilyOptions); + assertThat(columnFamilyOptions.setBlobGarbageCollectionForceThreshold(0.80)) + .isEqualTo(columnFamilyOptions); assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(true); assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(132768L); @@ -123,6 +129,7 @@ public class BlobOptionsTest { assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(true); assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(132768L); assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); + assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); } } @@ -135,6 +142,7 @@ public class BlobOptionsTest { .setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION) .setEnableBlobGarbageCollection(true) .setBlobGarbageCollectionAgeCutoff(0.89) + .setBlobGarbageCollectionForceThreshold(0.80) .setBlobFileSize(132768); assertThat(builder.enableBlobFiles()).isEqualTo(true); @@ -142,6 +150,7 @@ public class BlobOptionsTest { assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION); assertThat(builder.enableBlobGarbageCollection()).isEqualTo(true); assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); + assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); assertThat(builder.blobFileSize()).isEqualTo(132768); builder.setEnableBlobFiles(false) @@ -149,6 +158,7 @@ public class BlobOptionsTest { .setBlobCompressionType(CompressionType.LZ4_COMPRESSION) .setEnableBlobGarbageCollection(false) .setBlobGarbageCollectionAgeCutoff(0.91) + .setBlobGarbageCollectionForceThreshold(0.96) .setBlobFileSize(2048); assertThat(builder.enableBlobFiles()).isEqualTo(false); @@ -156,15 +166,17 @@ public class BlobOptionsTest { assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.LZ4_COMPRESSION); assertThat(builder.enableBlobGarbageCollection()).isEqualTo(false); assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.91); + assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.96); assertThat(builder.blobFileSize()).isEqualTo(2048); final MutableColumnFamilyOptions options = builder.build(); assertThat(options.getKeys()) .isEqualTo(new String[] {"enable_blob_files", "min_blob_size", "blob_compression_type", "enable_blob_garbage_collection", "blob_garbage_collection_age_cutoff", - "blob_file_size"}); + "blob_garbage_collection_force_threshold", "blob_file_size"}); assertThat(options.getValues()) - .isEqualTo(new String[] {"false", "4096", "LZ4_COMPRESSION", "false", "0.91", "2048"}); + .isEqualTo( + new String[] {"false", "4096", "LZ4_COMPRESSION", "false", "0.91", "0.96", "2048"}); } /** diff --git a/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java b/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java index baa42e786..9cbec5dd1 100644 --- a/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java +++ b/java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java @@ -96,7 +96,7 @@ public class MutableColumnFamilyOptionsTest { public void mutableColumnFamilyOptions_parse_getOptions_output() { final String optionsString = "bottommost_compression=kDisableCompressionOption; sample_for_compression=0; " - + "blob_garbage_collection_age_cutoff=0.250000; arena_block_size=1048576; enable_blob_garbage_collection=false; " + + "blob_garbage_collection_age_cutoff=0.250000; blob_garbage_collection_force_threshold=0.800000; arena_block_size=1048576; enable_blob_garbage_collection=false; " + "level0_stop_writes_trigger=36; min_blob_size=65536; " + "compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;" + "compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;}; " @@ -126,6 +126,7 @@ public class MutableColumnFamilyOptionsTest { // Check the values from the parsed string which are column family options assertThat(cf.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(cf.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); assertThat(cf.arenaBlockSize()).isEqualTo(1048576); assertThat(cf.enableBlobGarbageCollection()).isEqualTo(false); assertThat(cf.level0StopWritesTrigger()).isEqualTo(36); diff --git a/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java b/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java index 48bc660fe..95b1209e5 100644 --- a/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java +++ b/java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java @@ -44,6 +44,8 @@ public class MutableOptionsGetSetTest { new ColumnFamilyOptions() .setMinBlobSize(minBlobSize) .setEnableBlobFiles(true) + .setBlobGarbageCollectionAgeCutoff(0.25) + .setBlobGarbageCollectionForceThreshold(0.80) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.17) .setMemtableHugePageSize(3) @@ -98,6 +100,8 @@ public class MutableOptionsGetSetTest { final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = db.getOptions(columnFamilyHandle1); assertThat(builder1.enableBlobFiles()).isEqualTo(true); + assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder1.arenaBlockSize()).isEqualTo(42); assertThat(builder1.memtableHugePageSize()).isEqualTo(3); @@ -184,6 +188,8 @@ public class MutableOptionsGetSetTest { MutableColumnFamilyOptions.builder() .setMinBlobSize(minBlobSize) .setEnableBlobFiles(true) + .setBlobGarbageCollectionAgeCutoff(0.25) + .setBlobGarbageCollectionForceThreshold(0.80) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.17) .setMemtableHugePageSize(3) @@ -205,6 +211,8 @@ public class MutableOptionsGetSetTest { final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = db.getOptions(columnFamilyHandle1); assertThat(builder1.enableBlobFiles()).isEqualTo(true); + assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder1.arenaBlockSize()).isEqualTo(42); assertThat(builder1.memtableHugePageSize()).isEqualTo(3); @@ -294,6 +302,8 @@ public class MutableOptionsGetSetTest { MutableColumnFamilyOptions.builder() .setMinBlobSize(minBlobSize) .setEnableBlobFiles(true) + .setBlobGarbageCollectionAgeCutoff(0.25) + .setBlobGarbageCollectionForceThreshold(0.80) .setArenaBlockSize(42) .setMemtablePrefixBloomSizeRatio(0.17) .setMemtableHugePageSize(3) @@ -314,6 +324,8 @@ public class MutableOptionsGetSetTest { // Check the getOptions() brings back the creation options for CF1 final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = db.getOptions(); assertThat(builder1.enableBlobFiles()).isEqualTo(true); + assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); + assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80); assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); assertThat(builder1.arenaBlockSize()).isEqualTo(42); assertThat(builder1.memtableHugePageSize()).isEqualTo(3);