NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Java wrapper for blob_gc_force_threshold as blobGarbageCollectionForceThreshold (#9109)

Browse Source 
Summary:
Extra option added as a supplement to https://github.com/facebook/rocksdb/pull/8999

Closes https://github.com/facebook/rocksdb/issues/8221

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9109

Reviewed By: mrambacher

Differential Revision: D32065039

Pulled By: ltamasi

fbshipit-source-id: 6c484050a30fe0523850a8a3c95dc85b0a501362
main
Alan Paxton 3 years ago committed by Facebook GitHub Bot
parent 2b70224f82
commit 73e6b89fad
8 changed files with 171 additions and 6 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 51
      java/rocksjni/options.cc
  2. 25
      java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java
  3. 36
      java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
  4. 15
      java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
  5. 19
      java/src/main/java/org/rocksdb/Options.java
  6. 16
      java/src/test/java/org/rocksdb/BlobOptionsTest.java
  7. 3
      java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java
  8. 12
      java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java

51
java/rocksjni/options.cc
Unescape View File

@ -3837,6 +3837,30 @@ jdouble Java_org_rocksdb_Options_blobGarbageCollectionAgeCutoff(JNIEnv*,
return static_cast<jdouble>(opts->blob_garbage_collection_age_cutoff);
}
/*
* Class: org_rocksdb_Options
* Method: setBlobGarbageCollectionForceThreshold
* Signature: (JD)V
*/
void Java_org_rocksdb_Options_setBlobGarbageCollectionForceThreshold(
JNIEnv*, jobject, jlong jhandle,
jdouble jblob_garbage_collection_force_threshold) {
auto* opts = reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle);
opts->blob_garbage_collection_force_threshold =
static_cast<double>(jblob_garbage_collection_force_threshold);
}
/*
* Class: org_rocksdb_Options
* Method: blobGarbageCollectionForceThreshold
* Signature: (J)D
*/
jdouble Java_org_rocksdb_Options_blobGarbageCollectionForceThreshold(
JNIEnv*, jobject, jlong jhandle) {
auto* opts = reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle);
return static_cast<jdouble>(opts->blob_garbage_collection_force_threshold);
}
//////////////////////////////////////////////////////////////////////////////
// ROCKSDB_NAMESPACE::ColumnFamilyOptions
@ -5545,6 +5569,33 @@ jdouble Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionAgeCutoff(
return static_cast<jdouble>(opts->blob_garbage_collection_age_cutoff);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setBlobGarbageCollectionForceThreshold
* Signature: (JD)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setBlobGarbageCollectionForceThreshold(
JNIEnv*, jobject, jlong jhandle,
jdouble jblob_garbage_collection_force_threshold) {
auto* opts =
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle);
opts->blob_garbage_collection_force_threshold =
static_cast<double>(jblob_garbage_collection_force_threshold);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: blobGarbageCollectionAgeCutoff
* Signature: (J)D
*/
jdouble
Java_org_rocksdb_ColumnFamilyOptions_blobGarbageCollectionForceThreshold(
JNIEnv*, jobject, jlong jhandle) {
auto* opts =
reinterpret_cast<ROCKSDB_NAMESPACE::ColumnFamilyOptions*>(jhandle);
return static_cast<jdouble>(opts->blob_garbage_collection_force_threshold);
}
/////////////////////////////////////////////////////////////////////
// ROCKSDB_NAMESPACE::DBOptions

25
java/src/main/java/org/rocksdb/AdvancedMutableColumnFamilyOptionsInterface.java
Unescape View File

@ -687,6 +687,31 @@ public interface AdvancedMutableColumnFamilyOptionsInterface<
*/
double blobGarbageCollectionAgeCutoff();
/**
* If the ratio of garbage in the oldest blob files exceeds this threshold,
* targeted compactions are scheduled in order to force garbage collecting
* the blob files in question, assuming they are all eligible based on the
* value of {@link #blobGarbageCollectionAgeCutoff} above. This option is
* currently only supported with leveled compactions.
*
* Note that {@link #enableBlobGarbageCollection} has to be set in order for this
* option to have any effect.
*
* Default: 1.0
*
* Dynamically changeable through the SetOptions() API
*
* @param blobGarbageCollectionForceThreshold new value for the threshold
* @return the reference to the current options
*/
T setBlobGarbageCollectionForceThreshold(double blobGarbageCollectionForceThreshold);
/**
* Get the current value for the {@link #blobGarbageCollectionForceThreshold}
* @return the current threshold at which garbage collection of blobs is forced
*/
double blobGarbageCollectionForceThreshold();
//
// END options for blobs (integrated BlobDB)
//

36
java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
Unescape View File

@ -1164,6 +1164,39 @@ public class ColumnFamilyOptions extends RocksObject
return blobGarbageCollectionAgeCutoff(nativeHandle_);
}
/**
* If the ratio of garbage in the oldest blob files exceeds this threshold,
* targeted compactions are scheduled in order to force garbage collecting
* the blob files in question, assuming they are all eligible based on the
* value of {@link #blobGarbageCollectionAgeCutoff} above. This option is
* currently only supported with leveled compactions.
*
* Note that {@link #enableBlobGarbageCollection} has to be set in order for this
* option to have any effect.
*
* Default: 1.0
*
* Dynamically changeable through the SetOptions() API
*
* @param blobGarbageCollectionForceThreshold new value for the threshold
* @return the reference to the current options
*/
@Override
public ColumnFamilyOptions setBlobGarbageCollectionForceThreshold(
final double blobGarbageCollectionForceThreshold) {
setBlobGarbageCollectionForceThreshold(nativeHandle_, blobGarbageCollectionForceThreshold);
return this;
}
/**
* Get the current value for the {@link #blobGarbageCollectionForceThreshold}
* @return the current threshold at which garbage collection of blobs is forced
*/
@Override
public double blobGarbageCollectionForceThreshold() {
return blobGarbageCollectionForceThreshold(nativeHandle_);
}
//
// END options for blobs (integrated BlobDB)
//
@ -1358,6 +1391,9 @@ public class ColumnFamilyOptions extends RocksObject
private native void setBlobGarbageCollectionAgeCutoff(
final long nativeHandle_, final double blobGarbageCollectionAgeCutoff);
private native double blobGarbageCollectionAgeCutoff(final long nativeHandle_);
private native void setBlobGarbageCollectionForceThreshold(
final long nativeHandle_, final double blobGarbageCollectionForceThreshold);
private native double blobGarbageCollectionForceThreshold(final long nativeHandle_);
// instance variables
// NOTE: If you add new member variables, please update the copy constructor above!

15
java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
Unescape View File

@ -119,7 +119,8 @@ public class MutableColumnFamilyOptions
blob_file_size(ValueType.LONG),
blob_compression_type(ValueType.ENUM),
enable_blob_garbage_collection(ValueType.BOOLEAN),
blob_garbage_collection_age_cutoff(ValueType.DOUBLE);
blob_garbage_collection_age_cutoff(ValueType.DOUBLE),
blob_garbage_collection_force_threshold(ValueType.DOUBLE);
private final ValueType valueType;
BlobOption(final ValueType valueType) {
@ -546,5 +547,17 @@ public class MutableColumnFamilyOptions
public double blobGarbageCollectionAgeCutoff() {
return getDouble(BlobOption.blob_garbage_collection_age_cutoff);
}
@Override
public MutableColumnFamilyOptionsBuilder setBlobGarbageCollectionForceThreshold(
final double blobGarbageCollectionForceThreshold) {
return setDouble(
BlobOption.blob_garbage_collection_force_threshold, blobGarbageCollectionForceThreshold);
}
@Override
public double blobGarbageCollectionForceThreshold() {
return getDouble(BlobOption.blob_garbage_collection_force_threshold);
}
}
}

19
java/src/main/java/org/rocksdb/Options.java
Unescape View File

@ -2077,7 +2077,7 @@ public class Options extends RocksObject
}
@Override
public Options setBlobGarbageCollectionAgeCutoff(double blobGarbageCollectionAgeCutoff) {
public Options setBlobGarbageCollectionAgeCutoff(final double blobGarbageCollectionAgeCutoff) {
setBlobGarbageCollectionAgeCutoff(nativeHandle_, blobGarbageCollectionAgeCutoff);
return this;
}
@ -2087,6 +2087,18 @@ public class Options extends RocksObject
return blobGarbageCollectionAgeCutoff(nativeHandle_);
}
@Override
public Options setBlobGarbageCollectionForceThreshold(
final double blobGarbageCollectionForceThreshold) {
setBlobGarbageCollectionForceThreshold(nativeHandle_, blobGarbageCollectionForceThreshold);
return this;
}
@Override
public double blobGarbageCollectionForceThreshold() {
return blobGarbageCollectionForceThreshold(nativeHandle_);
}
//
// END options for blobs (integrated BlobDB)
//
@ -2517,8 +2529,11 @@ public class Options extends RocksObject
final long nativeHandle_, final boolean enableBlobGarbageCollection);
private native boolean enableBlobGarbageCollection(final long nativeHandle_);
private native void setBlobGarbageCollectionAgeCutoff(
final long nativeHandle_, double blobGarbageCollectionAgeCutoff);
final long nativeHandle_, final double blobGarbageCollectionAgeCutoff);
private native double blobGarbageCollectionAgeCutoff(final long nativeHandle_);
private native void setBlobGarbageCollectionForceThreshold(
final long nativeHandle_, final double blobGarbageCollectionForceThreshold);
private native double blobGarbageCollectionForceThreshold(final long nativeHandle_);
// instance variables
// NOTE: If you add new member variables, please update the copy constructor above!

16
java/src/test/java/org/rocksdb/BlobOptionsTest.java
Unescape View File

@ -77,6 +77,7 @@ public class BlobOptionsTest {
assertThat(options.enableBlobGarbageCollection()).isEqualTo(false);
assertThat(options.blobFileSize()).isEqualTo(268435456L);
assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(1.0);
assertThat(options.setEnableBlobFiles(true)).isEqualTo(options);
assertThat(options.setMinBlobSize(132768L)).isEqualTo(options);
@ -85,6 +86,7 @@ public class BlobOptionsTest {
assertThat(options.setEnableBlobGarbageCollection(true)).isEqualTo(options);
assertThat(options.setBlobFileSize(132768L)).isEqualTo(options);
assertThat(options.setBlobGarbageCollectionAgeCutoff(0.89)).isEqualTo(options);
assertThat(options.setBlobGarbageCollectionForceThreshold(0.80)).isEqualTo(options);
assertThat(options.enableBlobFiles()).isEqualTo(true);
assertThat(options.minBlobSize()).isEqualTo(132768L);
@ -92,6 +94,7 @@ public class BlobOptionsTest {
assertThat(options.enableBlobGarbageCollection()).isEqualTo(true);
assertThat(options.blobFileSize()).isEqualTo(132768L);
assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89);
assertThat(options.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
}
}
@ -105,6 +108,7 @@ public class BlobOptionsTest {
assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(false);
assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(268435456L);
assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(1.0);
assertThat(columnFamilyOptions.setEnableBlobFiles(true)).isEqualTo(columnFamilyOptions);
assertThat(columnFamilyOptions.setMinBlobSize(132768L)).isEqualTo(columnFamilyOptions);
@ -115,6 +119,8 @@ public class BlobOptionsTest {
assertThat(columnFamilyOptions.setBlobFileSize(132768L)).isEqualTo(columnFamilyOptions);
assertThat(columnFamilyOptions.setBlobGarbageCollectionAgeCutoff(0.89))
.isEqualTo(columnFamilyOptions);
assertThat(columnFamilyOptions.setBlobGarbageCollectionForceThreshold(0.80))
.isEqualTo(columnFamilyOptions);
assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(true);
assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(132768L);
@ -123,6 +129,7 @@ public class BlobOptionsTest {
assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(true);
assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(132768L);
assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89);
assertThat(columnFamilyOptions.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
}
}
@ -135,6 +142,7 @@ public class BlobOptionsTest {
.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION)
.setEnableBlobGarbageCollection(true)
.setBlobGarbageCollectionAgeCutoff(0.89)
.setBlobGarbageCollectionForceThreshold(0.80)
.setBlobFileSize(132768);
assertThat(builder.enableBlobFiles()).isEqualTo(true);
@ -142,6 +150,7 @@ public class BlobOptionsTest {
assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION);
assertThat(builder.enableBlobGarbageCollection()).isEqualTo(true);
assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89);
assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder.blobFileSize()).isEqualTo(132768);
builder.setEnableBlobFiles(false)
@ -149,6 +158,7 @@ public class BlobOptionsTest {
.setBlobCompressionType(CompressionType.LZ4_COMPRESSION)
.setEnableBlobGarbageCollection(false)
.setBlobGarbageCollectionAgeCutoff(0.91)
.setBlobGarbageCollectionForceThreshold(0.96)
.setBlobFileSize(2048);
assertThat(builder.enableBlobFiles()).isEqualTo(false);
@ -156,15 +166,17 @@ public class BlobOptionsTest {
assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.LZ4_COMPRESSION);
assertThat(builder.enableBlobGarbageCollection()).isEqualTo(false);
assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.91);
assertThat(builder.blobGarbageCollectionForceThreshold()).isEqualTo(0.96);
assertThat(builder.blobFileSize()).isEqualTo(2048);
final MutableColumnFamilyOptions options = builder.build();
assertThat(options.getKeys())
.isEqualTo(new String[] {"enable_blob_files", "min_blob_size", "blob_compression_type",
"enable_blob_garbage_collection", "blob_garbage_collection_age_cutoff",
"blob_file_size"});
"blob_garbage_collection_force_threshold", "blob_file_size"});
assertThat(options.getValues())
.isEqualTo(new String[] {"false", "4096", "LZ4_COMPRESSION", "false", "0.91", "2048"});
.isEqualTo(
new String[] {"false", "4096", "LZ4_COMPRESSION", "false", "0.91", "0.96", "2048"});
}
/**

3
java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java
Unescape View File

@ -96,7 +96,7 @@ public class MutableColumnFamilyOptionsTest {
public void mutableColumnFamilyOptions_parse_getOptions_output() {
final String optionsString =
"bottommost_compression=kDisableCompressionOption; sample_for_compression=0; "
+ "blob_garbage_collection_age_cutoff=0.250000; arena_block_size=1048576; enable_blob_garbage_collection=false; "
+ "blob_garbage_collection_age_cutoff=0.250000; blob_garbage_collection_force_threshold=0.800000; arena_block_size=1048576; enable_blob_garbage_collection=false; "
+ "level0_stop_writes_trigger=36; min_blob_size=65536; "
+ "compaction_options_universal={allow_trivial_move=false;stop_style=kCompactionStopStyleTotalSize;min_merge_width=2;"
+ "compression_size_percent=-1;max_size_amplification_percent=200;max_merge_width=4294967295;size_ratio=1;}; "
@ -126,6 +126,7 @@ public class MutableColumnFamilyOptionsTest {
// Check the values from the parsed string which are column family options
assertThat(cf.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(cf.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(cf.arenaBlockSize()).isEqualTo(1048576);
assertThat(cf.enableBlobGarbageCollection()).isEqualTo(false);
assertThat(cf.level0StopWritesTrigger()).isEqualTo(36);

12
java/src/test/java/org/rocksdb/MutableOptionsGetSetTest.java
Unescape View File

@ -44,6 +44,8 @@ public class MutableOptionsGetSetTest {
new ColumnFamilyOptions()
.setMinBlobSize(minBlobSize)
.setEnableBlobFiles(true)
.setBlobGarbageCollectionAgeCutoff(0.25)
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableHugePageSize(3)
@ -98,6 +100,8 @@ public class MutableOptionsGetSetTest {
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 =
db.getOptions(columnFamilyHandle1);
assertThat(builder1.enableBlobFiles()).isEqualTo(true);
assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
@ -184,6 +188,8 @@ public class MutableOptionsGetSetTest {
MutableColumnFamilyOptions.builder()
.setMinBlobSize(minBlobSize)
.setEnableBlobFiles(true)
.setBlobGarbageCollectionAgeCutoff(0.25)
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableHugePageSize(3)
@ -205,6 +211,8 @@ public class MutableOptionsGetSetTest {
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 =
db.getOptions(columnFamilyHandle1);
assertThat(builder1.enableBlobFiles()).isEqualTo(true);
assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);
@ -294,6 +302,8 @@ public class MutableOptionsGetSetTest {
MutableColumnFamilyOptions.builder()
.setMinBlobSize(minBlobSize)
.setEnableBlobFiles(true)
.setBlobGarbageCollectionAgeCutoff(0.25)
.setBlobGarbageCollectionForceThreshold(0.80)
.setArenaBlockSize(42)
.setMemtablePrefixBloomSizeRatio(0.17)
.setMemtableHugePageSize(3)
@ -314,6 +324,8 @@ public class MutableOptionsGetSetTest {
// Check the getOptions() brings back the creation options for CF1
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = db.getOptions();
assertThat(builder1.enableBlobFiles()).isEqualTo(true);
assertThat(builder1.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25);
assertThat(builder1.blobGarbageCollectionForceThreshold()).isEqualTo(0.80);
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize);
assertThat(builder1.arenaBlockSize()).isEqualTo(42);
assertThat(builder1.memtableHugePageSize()).isEqualTo(3);

Write Preview
Loading…
Cancel
Save