New-style blob option bindings, Java option getter and improve/fix option parsing (#8999)
Summary: Implementation of https://github.com/facebook/rocksdb/issues/8221, plus/including extension of Java options API to allow the get() of options from RocksDB. The extension allows more comprehensive testing of options at the Java side, by validating that the options are set at the C++ side. Variations on methods: MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder getOptions() MutableDBOptions.MutableDBOptionsBuilder getDBOptions() retrieve the options via RocksDB C++ interfaces, and parse the resulting string into one of the Java-style option objects. This necessitated generalising the parsing of option strings in Java, which now parses the full range of option strings returned by the C++ interface, rather than a useful subset. This necessitates the list-separator being changed to :(colon) from , (comma). Pull Request resolved: https://github.com/facebook/rocksdb/pull/8999 Reviewed By: jay-zhuang Differential Revision: D31655487 Pulled By: ltamasi fbshipit-source-id: c38e98145c81c61dc38238b0df580db176ce4efdmain
Alan Paxton
3 years ago
committed by
Facebook GitHub Bot
parent
ad5325a736
commit
8d615a2b1d
@ -0,0 +1,256 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
package org.rocksdb; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
import java.util.Objects; |
||||
|
||||
public class OptionString { |
||||
private final static char kvPairSeparator = ';'; |
||||
private final static char kvSeparator = '='; |
||||
private final static char complexValueBegin = '{'; |
||||
private final static char complexValueEnd = '}'; |
||||
private final static char wrappedValueBegin = '{'; |
||||
private final static char wrappedValueEnd = '}'; |
||||
private final static char arrayValueSeparator = ':'; |
||||
|
||||
static class Value { |
||||
final List<String> list; |
||||
final List<Entry> complex; |
||||
|
||||
public Value(final List<String> list, final List<Entry> complex) { |
||||
this.list = list; |
||||
this.complex = complex; |
||||
} |
||||
|
||||
public boolean isList() { |
||||
return (this.list != null && this.complex == null); |
||||
} |
||||
|
||||
public static Value fromList(final List<String> list) { |
||||
return new Value(list, null); |
||||
} |
||||
|
||||
public static Value fromComplex(final List<Entry> complex) { |
||||
return new Value(null, complex); |
||||
} |
||||
|
||||
public String toString() { |
||||
final StringBuilder sb = new StringBuilder(); |
||||
if (isList()) { |
||||
for (final String item : list) { |
||||
sb.append(item).append(arrayValueSeparator); |
||||
} |
||||
// remove the final separator
|
||||
if (sb.length() > 0) |
||||
sb.delete(sb.length() - 1, sb.length()); |
||||
} else { |
||||
sb.append('['); |
||||
for (final Entry entry : complex) { |
||||
sb.append(entry.toString()).append(';'); |
||||
} |
||||
sb.append(']'); |
||||
} |
||||
return sb.toString(); |
||||
} |
||||
} |
||||
|
||||
static class Entry { |
||||
public final String key; |
||||
public final Value value; |
||||
|
||||
private Entry(final String key, final Value value) { |
||||
this.key = key; |
||||
this.value = value; |
||||
} |
||||
|
||||
public String toString() { |
||||
return "" + key + "=" + value; |
||||
} |
||||
} |
||||
|
||||
static class Parser { |
||||
static class Exception extends RuntimeException { |
||||
public Exception(final String s) { |
||||
super(s); |
||||
} |
||||
} |
||||
|
||||
final String str; |
||||
final StringBuilder sb; |
||||
|
||||
private Parser(final String str) { |
||||
this.str = str; |
||||
this.sb = new StringBuilder(str); |
||||
} |
||||
|
||||
private void exception(final String message) { |
||||
final int pos = str.length() - sb.length(); |
||||
final int before = Math.min(pos, 64); |
||||
final int after = Math.min(64, str.length() - pos); |
||||
final String here = |
||||
str.substring(pos - before, pos) + "__*HERE*__" + str.substring(pos, pos + after); |
||||
|
||||
throw new Parser.Exception(message + " at [" + here + "]"); |
||||
} |
||||
|
||||
private void skipWhite() { |
||||
while (sb.length() > 0 && Character.isWhitespace(sb.charAt(0))) { |
||||
sb.delete(0, 1); |
||||
} |
||||
} |
||||
|
||||
private char first() { |
||||
if (sb.length() == 0) |
||||
exception("Unexpected end of input"); |
||||
return sb.charAt(0); |
||||
} |
||||
|
||||
private char next() { |
||||
if (sb.length() == 0) |
||||
exception("Unexpected end of input"); |
||||
final char c = sb.charAt(0); |
||||
sb.delete(0, 1); |
||||
return c; |
||||
} |
||||
|
||||
private boolean hasNext() { |
||||
return (sb.length() > 0); |
||||
} |
||||
|
||||
private boolean is(final char c) { |
||||
return (sb.length() > 0 && sb.charAt(0) == c); |
||||
} |
||||
|
||||
private boolean isKeyChar() { |
||||
if (!hasNext()) |
||||
return false; |
||||
final char c = first(); |
||||
return (Character.isAlphabetic(c) || Character.isDigit(c) || "_".indexOf(c) != -1); |
||||
} |
||||
|
||||
private boolean isValueChar() { |
||||
if (!hasNext()) |
||||
return false; |
||||
final char c = first(); |
||||
return (Character.isAlphabetic(c) || Character.isDigit(c) || "_-+.[]".indexOf(c) != -1); |
||||
} |
||||
|
||||
private String parseKey() { |
||||
final StringBuilder sbKey = new StringBuilder(); |
||||
sbKey.append(next()); |
||||
while (isKeyChar()) { |
||||
sbKey.append(next()); |
||||
} |
||||
|
||||
return sbKey.toString(); |
||||
} |
||||
|
||||
private String parseSimpleValue() { |
||||
if (is(wrappedValueBegin)) { |
||||
next(); |
||||
final String result = parseSimpleValue(); |
||||
if (!is(wrappedValueEnd)) { |
||||
exception("Expected to end a wrapped value with " + wrappedValueEnd); |
||||
} |
||||
next(); |
||||
|
||||
return result; |
||||
} else { |
||||
final StringBuilder sbValue = new StringBuilder(); |
||||
while (isValueChar()) sbValue.append(next()); |
||||
|
||||
return sbValue.toString(); |
||||
} |
||||
} |
||||
|
||||
private List<String> parseList() { |
||||
final List<String> list = new ArrayList<>(1); |
||||
while (true) { |
||||
list.add(parseSimpleValue()); |
||||
if (!is(arrayValueSeparator)) |
||||
break; |
||||
|
||||
next(); |
||||
} |
||||
|
||||
return list; |
||||
} |
||||
|
||||
private Entry parseOption() { |
||||
skipWhite(); |
||||
if (!isKeyChar()) { |
||||
exception("No valid key character(s) for key in key=value "); |
||||
} |
||||
final String key = parseKey(); |
||||
skipWhite(); |
||||
if (is(kvSeparator)) { |
||||
next(); |
||||
} else { |
||||
exception("Expected = separating key and value"); |
||||
} |
||||
skipWhite(); |
||||
final Value value = parseValue(); |
||||
return new Entry(key, value); |
||||
} |
||||
|
||||
private Value parseValue() { |
||||
skipWhite(); |
||||
if (is(complexValueBegin)) { |
||||
next(); |
||||
skipWhite(); |
||||
final Value value = Value.fromComplex(parseComplex()); |
||||
skipWhite(); |
||||
if (is(complexValueEnd)) { |
||||
next(); |
||||
skipWhite(); |
||||
} else { |
||||
exception("Expected } ending complex value"); |
||||
} |
||||
return value; |
||||
} else if (isValueChar()) { |
||||
return Value.fromList(parseList()); |
||||
} |
||||
|
||||
exception("No valid value character(s) for value in key=value"); |
||||
return null; |
||||
} |
||||
|
||||
private List<Entry> parseComplex() { |
||||
final List<Entry> entries = new ArrayList<>(); |
||||
|
||||
skipWhite(); |
||||
if (hasNext()) { |
||||
entries.add(parseOption()); |
||||
skipWhite(); |
||||
while (is(kvPairSeparator)) { |
||||
next(); |
||||
skipWhite(); |
||||
if (!isKeyChar()) { |
||||
// the separator was a terminator
|
||||
break; |
||||
} |
||||
entries.add(parseOption()); |
||||
skipWhite(); |
||||
} |
||||
} |
||||
return entries; |
||||
} |
||||
|
||||
public static List<Entry> parse(final String str) { |
||||
Objects.requireNonNull(str); |
||||
|
||||
final Parser parser = new Parser(str); |
||||
final List<Entry> result = parser.parseComplex(); |
||||
if (parser.hasNext()) { |
||||
parser.exception("Unexpected end of parsing "); |
||||
} |
||||
|
||||
return result; |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,301 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
package org.rocksdb; |
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8; |
||||
import static org.assertj.core.api.Assertions.assertThat; |
||||
|
||||
import java.io.File; |
||||
import java.io.FilenameFilter; |
||||
import java.util.*; |
||||
import org.junit.ClassRule; |
||||
import org.junit.Rule; |
||||
import org.junit.Test; |
||||
import org.junit.rules.TemporaryFolder; |
||||
|
||||
public class BlobOptionsTest { |
||||
@ClassRule |
||||
public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = |
||||
new RocksNativeLibraryResource(); |
||||
|
||||
@Rule public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||
|
||||
final int minBlobSize = 65536; |
||||
final int largeBlobSize = 65536 * 2; |
||||
|
||||
/** |
||||
* Count the files in the temporary folder which end with a particular suffix |
||||
* Used to query the state of a test database to check if it is as the test expects |
||||
* |
||||
* @param endsWith the suffix to match |
||||
* @return the number of files with a matching suffix |
||||
*/ |
||||
@SuppressWarnings("CallToStringConcatCanBeReplacedByOperator") |
||||
private int countDBFiles(final String endsWith) { |
||||
return Objects |
||||
.requireNonNull(dbFolder.getRoot().list(new FilenameFilter() { |
||||
@Override |
||||
public boolean accept(File dir, String name) { |
||||
return name.endsWith(endsWith); |
||||
} |
||||
})) |
||||
.length; |
||||
} |
||||
|
||||
@SuppressWarnings("SameParameterValue") |
||||
private byte[] small_key(String suffix) { |
||||
return ("small_key_" + suffix).getBytes(UTF_8); |
||||
} |
||||
|
||||
@SuppressWarnings("SameParameterValue") |
||||
private byte[] small_value(String suffix) { |
||||
return ("small_value_" + suffix).getBytes(UTF_8); |
||||
} |
||||
|
||||
private byte[] large_key(String suffix) { |
||||
return ("large_key_" + suffix).getBytes(UTF_8); |
||||
} |
||||
|
||||
private byte[] large_value(String repeat) { |
||||
final byte[] large_value = ("" + repeat + "_" + largeBlobSize + "b").getBytes(UTF_8); |
||||
final byte[] large_buffer = new byte[largeBlobSize]; |
||||
for (int pos = 0; pos < largeBlobSize; pos += large_value.length) { |
||||
int numBytes = Math.min(large_value.length, large_buffer.length - pos); |
||||
System.arraycopy(large_value, 0, large_buffer, pos, numBytes); |
||||
} |
||||
return large_buffer; |
||||
} |
||||
|
||||
@Test |
||||
public void blobOptions() { |
||||
try (final Options options = new Options()) { |
||||
assertThat(options.enableBlobFiles()).isEqualTo(false); |
||||
assertThat(options.minBlobSize()).isEqualTo(0); |
||||
assertThat(options.blobCompressionType()).isEqualTo(CompressionType.NO_COMPRESSION); |
||||
assertThat(options.enableBlobGarbageCollection()).isEqualTo(false); |
||||
assertThat(options.blobFileSize()).isEqualTo(268435456L); |
||||
assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); |
||||
|
||||
assertThat(options.setEnableBlobFiles(true)).isEqualTo(options); |
||||
assertThat(options.setMinBlobSize(132768L)).isEqualTo(options); |
||||
assertThat(options.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION)) |
||||
.isEqualTo(options); |
||||
assertThat(options.setEnableBlobGarbageCollection(true)).isEqualTo(options); |
||||
assertThat(options.setBlobFileSize(132768L)).isEqualTo(options); |
||||
assertThat(options.setBlobGarbageCollectionAgeCutoff(0.89)).isEqualTo(options); |
||||
|
||||
assertThat(options.enableBlobFiles()).isEqualTo(true); |
||||
assertThat(options.minBlobSize()).isEqualTo(132768L); |
||||
assertThat(options.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION); |
||||
assertThat(options.enableBlobGarbageCollection()).isEqualTo(true); |
||||
assertThat(options.blobFileSize()).isEqualTo(132768L); |
||||
assertThat(options.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void blobColumnFamilyOptions() { |
||||
try (final ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions()) { |
||||
assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(false); |
||||
assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(0); |
||||
assertThat(columnFamilyOptions.blobCompressionType()) |
||||
.isEqualTo(CompressionType.NO_COMPRESSION); |
||||
assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(false); |
||||
assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(268435456L); |
||||
assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.25); |
||||
|
||||
assertThat(columnFamilyOptions.setEnableBlobFiles(true)).isEqualTo(columnFamilyOptions); |
||||
assertThat(columnFamilyOptions.setMinBlobSize(132768L)).isEqualTo(columnFamilyOptions); |
||||
assertThat(columnFamilyOptions.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION)) |
||||
.isEqualTo(columnFamilyOptions); |
||||
assertThat(columnFamilyOptions.setEnableBlobGarbageCollection(true)) |
||||
.isEqualTo(columnFamilyOptions); |
||||
assertThat(columnFamilyOptions.setBlobFileSize(132768L)).isEqualTo(columnFamilyOptions); |
||||
assertThat(columnFamilyOptions.setBlobGarbageCollectionAgeCutoff(0.89)) |
||||
.isEqualTo(columnFamilyOptions); |
||||
|
||||
assertThat(columnFamilyOptions.enableBlobFiles()).isEqualTo(true); |
||||
assertThat(columnFamilyOptions.minBlobSize()).isEqualTo(132768L); |
||||
assertThat(columnFamilyOptions.blobCompressionType()) |
||||
.isEqualTo(CompressionType.BZLIB2_COMPRESSION); |
||||
assertThat(columnFamilyOptions.enableBlobGarbageCollection()).isEqualTo(true); |
||||
assertThat(columnFamilyOptions.blobFileSize()).isEqualTo(132768L); |
||||
assertThat(columnFamilyOptions.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void blobMutableColumnFamilyOptionsBuilder() { |
||||
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder = |
||||
MutableColumnFamilyOptions.builder(); |
||||
builder.setEnableBlobFiles(true) |
||||
.setMinBlobSize(1024) |
||||
.setBlobCompressionType(CompressionType.BZLIB2_COMPRESSION) |
||||
.setEnableBlobGarbageCollection(true) |
||||
.setBlobGarbageCollectionAgeCutoff(0.89) |
||||
.setBlobFileSize(132768); |
||||
|
||||
assertThat(builder.enableBlobFiles()).isEqualTo(true); |
||||
assertThat(builder.minBlobSize()).isEqualTo(1024); |
||||
assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.BZLIB2_COMPRESSION); |
||||
assertThat(builder.enableBlobGarbageCollection()).isEqualTo(true); |
||||
assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.89); |
||||
assertThat(builder.blobFileSize()).isEqualTo(132768); |
||||
|
||||
builder.setEnableBlobFiles(false) |
||||
.setMinBlobSize(4096) |
||||
.setBlobCompressionType(CompressionType.LZ4_COMPRESSION) |
||||
.setEnableBlobGarbageCollection(false) |
||||
.setBlobGarbageCollectionAgeCutoff(0.91) |
||||
.setBlobFileSize(2048); |
||||
|
||||
assertThat(builder.enableBlobFiles()).isEqualTo(false); |
||||
assertThat(builder.minBlobSize()).isEqualTo(4096); |
||||
assertThat(builder.blobCompressionType()).isEqualTo(CompressionType.LZ4_COMPRESSION); |
||||
assertThat(builder.enableBlobGarbageCollection()).isEqualTo(false); |
||||
assertThat(builder.blobGarbageCollectionAgeCutoff()).isEqualTo(0.91); |
||||
assertThat(builder.blobFileSize()).isEqualTo(2048); |
||||
|
||||
final MutableColumnFamilyOptions options = builder.build(); |
||||
assertThat(options.getKeys()) |
||||
.isEqualTo(new String[] {"enable_blob_files", "min_blob_size", "blob_compression_type", |
||||
"enable_blob_garbage_collection", "blob_garbage_collection_age_cutoff", |
||||
"blob_file_size"}); |
||||
assertThat(options.getValues()) |
||||
.isEqualTo(new String[] {"false", "4096", "LZ4_COMPRESSION", "false", "0.91", "2048"}); |
||||
} |
||||
|
||||
/** |
||||
* Configure the default column family with BLOBs. |
||||
* Confirm that BLOBs are generated when appropriately-sized writes are flushed. |
||||
* |
||||
* @throws RocksDBException if a db access throws an exception |
||||
*/ |
||||
@Test |
||||
public void testBlobWriteAboveThreshold() throws RocksDBException { |
||||
try (final Options options = new Options() |
||||
.setCreateIfMissing(true) |
||||
.setMinBlobSize(minBlobSize) |
||||
.setEnableBlobFiles(true); |
||||
|
||||
final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) { |
||||
db.put(small_key("default"), small_value("default")); |
||||
db.flush(new FlushOptions().setWaitForFlush(true)); |
||||
|
||||
// check there are no blobs in the database
|
||||
assertThat(countDBFiles(".sst")).isEqualTo(1); |
||||
assertThat(countDBFiles(".blob")).isEqualTo(0); |
||||
|
||||
db.put(large_key("default"), large_value("default")); |
||||
db.flush(new FlushOptions().setWaitForFlush(true)); |
||||
|
||||
// wrote and flushed a value larger than the blobbing threshold
|
||||
// check there is a single blob in the database
|
||||
assertThat(countDBFiles(".sst")).isEqualTo(2); |
||||
assertThat(countDBFiles(".blob")).isEqualTo(1); |
||||
|
||||
assertThat(db.get(small_key("default"))).isEqualTo(small_value("default")); |
||||
assertThat(db.get(large_key("default"))).isEqualTo(large_value("default")); |
||||
|
||||
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder fetchOptions = |
||||
db.getOptions(null); |
||||
assertThat(fetchOptions.minBlobSize()).isEqualTo(minBlobSize); |
||||
assertThat(fetchOptions.enableBlobFiles()).isEqualTo(true); |
||||
assertThat(fetchOptions.writeBufferSize()).isEqualTo(64 << 20); |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Configure 2 column families respectively with and without BLOBs. |
||||
* Confirm that BLOB files are generated (once the DB is flushed) only for the appropriate column |
||||
* family. |
||||
* |
||||
* @throws RocksDBException if a db access throws an exception |
||||
*/ |
||||
@Test |
||||
public void testBlobWriteAboveThresholdCF() throws RocksDBException { |
||||
final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor0 = |
||||
new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); |
||||
List<ColumnFamilyDescriptor> columnFamilyDescriptors = |
||||
Collections.singletonList(columnFamilyDescriptor0); |
||||
List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(); |
||||
|
||||
try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); |
||||
final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), |
||||
columnFamilyDescriptors, columnFamilyHandles)) { |
||||
db.put(columnFamilyHandles.get(0), small_key("default"), small_value("default")); |
||||
db.flush(new FlushOptions().setWaitForFlush(true)); |
||||
|
||||
assertThat(countDBFiles(".blob")).isEqualTo(0); |
||||
|
||||
try (final ColumnFamilyOptions columnFamilyOptions1 = |
||||
new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(true); |
||||
|
||||
final ColumnFamilyOptions columnFamilyOptions2 = |
||||
new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(false)) { |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor1 = |
||||
new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor2 = |
||||
new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); |
||||
|
||||
// Create the first column family with blob options
|
||||
db.createColumnFamily(columnFamilyDescriptor1); |
||||
|
||||
// Create the second column family with not-blob options
|
||||
db.createColumnFamily(columnFamilyDescriptor2); |
||||
} |
||||
} |
||||
|
||||
// Now re-open after auto-close - at this point the CF options we use are recognized.
|
||||
try (final ColumnFamilyOptions columnFamilyOptions1 = |
||||
new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(true); |
||||
|
||||
final ColumnFamilyOptions columnFamilyOptions2 = |
||||
new ColumnFamilyOptions().setMinBlobSize(minBlobSize).setEnableBlobFiles(false)) { |
||||
assertThat(columnFamilyOptions1.enableBlobFiles()).isEqualTo(true); |
||||
assertThat(columnFamilyOptions1.minBlobSize()).isEqualTo(minBlobSize); |
||||
assertThat(columnFamilyOptions2.enableBlobFiles()).isEqualTo(false); |
||||
assertThat(columnFamilyOptions1.minBlobSize()).isEqualTo(minBlobSize); |
||||
|
||||
final ColumnFamilyDescriptor columnFamilyDescriptor1 = |
||||
new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor2 = |
||||
new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); |
||||
columnFamilyDescriptors = new ArrayList<>(); |
||||
columnFamilyDescriptors.add(columnFamilyDescriptor0); |
||||
columnFamilyDescriptors.add(columnFamilyDescriptor1); |
||||
columnFamilyDescriptors.add(columnFamilyDescriptor2); |
||||
columnFamilyHandles = new ArrayList<>(); |
||||
|
||||
assertThat(columnFamilyDescriptor1.getOptions().enableBlobFiles()).isEqualTo(true); |
||||
assertThat(columnFamilyDescriptor2.getOptions().enableBlobFiles()).isEqualTo(false); |
||||
|
||||
try (final DBOptions dbOptions = new DBOptions(); |
||||
final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), |
||||
columnFamilyDescriptors, columnFamilyHandles)) { |
||||
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = |
||||
db.getOptions(columnFamilyHandles.get(1)); |
||||
assertThat(builder1.enableBlobFiles()).isEqualTo(true); |
||||
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); |
||||
|
||||
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder2 = |
||||
db.getOptions(columnFamilyHandles.get(2)); |
||||
assertThat(builder2.enableBlobFiles()).isEqualTo(false); |
||||
assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize); |
||||
|
||||
db.put(columnFamilyHandles.get(1), large_key("column_family_1_k2"), |
||||
large_value("column_family_1_k2")); |
||||
db.flush(new FlushOptions().setWaitForFlush(true), columnFamilyHandles.get(1)); |
||||
assertThat(countDBFiles(".blob")).isEqualTo(1); |
||||
|
||||
db.put(columnFamilyHandles.get(2), large_key("column_family_2_k2"), |
||||
large_value("column_family_2_k2")); |
||||
db.flush(new FlushOptions().setWaitForFlush(true), columnFamilyHandles.get(2)); |
||||
assertThat(countDBFiles(".blob")).isEqualTo(1); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,385 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
package org.rocksdb; |
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8; |
||||
import static org.assertj.core.api.Assertions.assertThat; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.Collections; |
||||
import java.util.List; |
||||
import org.junit.Rule; |
||||
import org.junit.Test; |
||||
import org.junit.rules.TemporaryFolder; |
||||
|
||||
public class MutableOptionsGetSetTest { |
||||
final int minBlobSize = 65536; |
||||
|
||||
@Rule public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||
|
||||
/** |
||||
* Validate the round-trip of blob options into and out of the C++ core of RocksDB |
||||
* From CF options on CF Creation to {RocksDB#getOptions} |
||||
* Uses 2x column families with different values for their options. |
||||
* NOTE that some constraints are applied to the options in the C++ core, |
||||
* e.g. on {ColumnFamilyOptions#setMemtablePrefixBloomSizeRatio} |
||||
* |
||||
* @throws RocksDBException if the database throws an exception |
||||
*/ |
||||
@Test |
||||
public void testGetMutableBlobOptionsAfterCreate() throws RocksDBException { |
||||
final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor0 = |
||||
new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); |
||||
final List<ColumnFamilyDescriptor> columnFamilyDescriptors = |
||||
Collections.singletonList(columnFamilyDescriptor0); |
||||
final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(); |
||||
|
||||
try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); |
||||
final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), |
||||
columnFamilyDescriptors, columnFamilyHandles)) { |
||||
try (final ColumnFamilyOptions columnFamilyOptions1 = |
||||
new ColumnFamilyOptions() |
||||
.setMinBlobSize(minBlobSize) |
||||
.setEnableBlobFiles(true) |
||||
.setArenaBlockSize(42) |
||||
.setMemtablePrefixBloomSizeRatio(0.17) |
||||
.setMemtableHugePageSize(3) |
||||
.setMaxSuccessiveMerges(4) |
||||
.setMaxWriteBufferNumber(12) |
||||
.setInplaceUpdateNumLocks(16) |
||||
.setDisableAutoCompactions(false) |
||||
.setSoftPendingCompactionBytesLimit(112) |
||||
.setHardPendingCompactionBytesLimit(280) |
||||
.setLevel0FileNumCompactionTrigger(200) |
||||
.setLevel0SlowdownWritesTrigger(312) |
||||
.setLevel0StopWritesTrigger(584) |
||||
.setMaxCompactionBytes(12) |
||||
.setTargetFileSizeBase(99) |
||||
.setTargetFileSizeMultiplier(112) |
||||
.setMaxSequentialSkipInIterations(50) |
||||
.setReportBgIoStats(true); |
||||
|
||||
final ColumnFamilyOptions columnFamilyOptions2 = |
||||
new ColumnFamilyOptions() |
||||
.setMinBlobSize(minBlobSize) |
||||
.setEnableBlobFiles(false) |
||||
.setArenaBlockSize(42) |
||||
.setMemtablePrefixBloomSizeRatio(0.236) |
||||
.setMemtableHugePageSize(8) |
||||
.setMaxSuccessiveMerges(12) |
||||
.setMaxWriteBufferNumber(22) |
||||
.setInplaceUpdateNumLocks(160) |
||||
.setDisableAutoCompactions(true) |
||||
.setSoftPendingCompactionBytesLimit(1124) |
||||
.setHardPendingCompactionBytesLimit(2800) |
||||
.setLevel0FileNumCompactionTrigger(2000) |
||||
.setLevel0SlowdownWritesTrigger(5840) |
||||
.setLevel0StopWritesTrigger(31200) |
||||
.setMaxCompactionBytes(112) |
||||
.setTargetFileSizeBase(999) |
||||
.setTargetFileSizeMultiplier(1120) |
||||
.setMaxSequentialSkipInIterations(24) |
||||
.setReportBgIoStats(true)) { |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor1 = |
||||
new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor2 = |
||||
new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); |
||||
|
||||
// Create the column family with blob options
|
||||
final ColumnFamilyHandle columnFamilyHandle1 = |
||||
db.createColumnFamily(columnFamilyDescriptor1); |
||||
final ColumnFamilyHandle columnFamilyHandle2 = |
||||
db.createColumnFamily(columnFamilyDescriptor2); |
||||
|
||||
// Check the getOptions() brings back the creation options for CF1
|
||||
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = |
||||
db.getOptions(columnFamilyHandle1); |
||||
assertThat(builder1.enableBlobFiles()).isEqualTo(true); |
||||
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); |
||||
assertThat(builder1.arenaBlockSize()).isEqualTo(42); |
||||
assertThat(builder1.memtableHugePageSize()).isEqualTo(3); |
||||
assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); |
||||
assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4); |
||||
assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12); |
||||
assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16); |
||||
assertThat(builder1.disableAutoCompactions()).isEqualTo(false); |
||||
assertThat(builder1.softPendingCompactionBytesLimit()).isEqualTo(112); |
||||
assertThat(builder1.hardPendingCompactionBytesLimit()).isEqualTo(280); |
||||
assertThat(builder1.level0FileNumCompactionTrigger()).isEqualTo(200); |
||||
assertThat(builder1.level0SlowdownWritesTrigger()).isEqualTo(312); |
||||
assertThat(builder1.level0StopWritesTrigger()).isEqualTo(584); |
||||
assertThat(builder1.maxCompactionBytes()).isEqualTo(12); |
||||
assertThat(builder1.targetFileSizeBase()).isEqualTo(99); |
||||
assertThat(builder1.targetFileSizeMultiplier()).isEqualTo(112); |
||||
assertThat(builder1.maxSequentialSkipInIterations()).isEqualTo(50); |
||||
assertThat(builder1.reportBgIoStats()).isEqualTo(true); |
||||
|
||||
// Check the getOptions() brings back the creation options for CF2
|
||||
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder2 = |
||||
db.getOptions(columnFamilyHandle2); |
||||
assertThat(builder2.enableBlobFiles()).isEqualTo(false); |
||||
assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize); |
||||
assertThat(builder2.arenaBlockSize()).isEqualTo(42); |
||||
assertThat(builder2.memtableHugePageSize()).isEqualTo(8); |
||||
assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236); |
||||
assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12); |
||||
assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22); |
||||
assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160); |
||||
assertThat(builder2.disableAutoCompactions()).isEqualTo(true); |
||||
assertThat(builder2.softPendingCompactionBytesLimit()).isEqualTo(1124); |
||||
assertThat(builder2.hardPendingCompactionBytesLimit()).isEqualTo(2800); |
||||
assertThat(builder2.level0FileNumCompactionTrigger()).isEqualTo(2000); |
||||
assertThat(builder2.level0SlowdownWritesTrigger()).isEqualTo(5840); |
||||
assertThat(builder2.level0StopWritesTrigger()).isEqualTo(31200); |
||||
assertThat(builder2.maxCompactionBytes()).isEqualTo(112); |
||||
assertThat(builder2.targetFileSizeBase()).isEqualTo(999); |
||||
assertThat(builder2.targetFileSizeMultiplier()).isEqualTo(1120); |
||||
assertThat(builder2.maxSequentialSkipInIterations()).isEqualTo(24); |
||||
assertThat(builder2.reportBgIoStats()).isEqualTo(true); |
||||
} |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Validate the round-trip of blob options into and out of the C++ core of RocksDB |
||||
* From {RocksDB#setOptions} to {RocksDB#getOptions} |
||||
* Uses 2x column families with different values for their options. |
||||
* NOTE that some constraints are applied to the options in the C++ core, |
||||
* e.g. on {ColumnFamilyOptions#setMemtablePrefixBloomSizeRatio} |
||||
* |
||||
* @throws RocksDBException if a database access has an error |
||||
*/ |
||||
@Test |
||||
public void testGetMutableBlobOptionsAfterSetCF() throws RocksDBException { |
||||
final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor0 = |
||||
new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); |
||||
final List<ColumnFamilyDescriptor> columnFamilyDescriptors = |
||||
Collections.singletonList(columnFamilyDescriptor0); |
||||
final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(); |
||||
|
||||
try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); |
||||
final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), |
||||
columnFamilyDescriptors, columnFamilyHandles)) { |
||||
try (final ColumnFamilyOptions columnFamilyOptions1 = new ColumnFamilyOptions(); |
||||
|
||||
final ColumnFamilyOptions columnFamilyOptions2 = new ColumnFamilyOptions()) { |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor1 = |
||||
new ColumnFamilyDescriptor("column_family_1".getBytes(UTF_8), columnFamilyOptions1); |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor2 = |
||||
new ColumnFamilyDescriptor("column_family_2".getBytes(UTF_8), columnFamilyOptions2); |
||||
|
||||
// Create the column family with blob options
|
||||
final ColumnFamilyHandle columnFamilyHandle1 = |
||||
db.createColumnFamily(columnFamilyDescriptor1); |
||||
final ColumnFamilyHandle columnFamilyHandle2 = |
||||
db.createColumnFamily(columnFamilyDescriptor2); |
||||
db.flush(new FlushOptions().setWaitForFlush(true)); |
||||
|
||||
final MutableColumnFamilyOptions |
||||
.MutableColumnFamilyOptionsBuilder mutableColumnFamilyOptions1 = |
||||
MutableColumnFamilyOptions.builder() |
||||
.setMinBlobSize(minBlobSize) |
||||
.setEnableBlobFiles(true) |
||||
.setArenaBlockSize(42) |
||||
.setMemtablePrefixBloomSizeRatio(0.17) |
||||
.setMemtableHugePageSize(3) |
||||
.setMaxSuccessiveMerges(4) |
||||
.setMaxWriteBufferNumber(12) |
||||
.setInplaceUpdateNumLocks(16) |
||||
.setDisableAutoCompactions(false) |
||||
.setSoftPendingCompactionBytesLimit(112) |
||||
.setHardPendingCompactionBytesLimit(280) |
||||
.setLevel0FileNumCompactionTrigger(200) |
||||
.setLevel0SlowdownWritesTrigger(312) |
||||
.setLevel0StopWritesTrigger(584) |
||||
.setMaxCompactionBytes(12) |
||||
.setTargetFileSizeBase(99) |
||||
.setTargetFileSizeMultiplier(112); |
||||
db.setOptions(columnFamilyHandle1, mutableColumnFamilyOptions1.build()); |
||||
|
||||
// Check the getOptions() brings back the creation options for CF1
|
||||
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = |
||||
db.getOptions(columnFamilyHandle1); |
||||
assertThat(builder1.enableBlobFiles()).isEqualTo(true); |
||||
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); |
||||
assertThat(builder1.arenaBlockSize()).isEqualTo(42); |
||||
assertThat(builder1.memtableHugePageSize()).isEqualTo(3); |
||||
assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); |
||||
assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4); |
||||
assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12); |
||||
assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16); |
||||
assertThat(builder1.disableAutoCompactions()).isEqualTo(false); |
||||
assertThat(builder1.softPendingCompactionBytesLimit()).isEqualTo(112); |
||||
assertThat(builder1.hardPendingCompactionBytesLimit()).isEqualTo(280); |
||||
assertThat(builder1.level0FileNumCompactionTrigger()).isEqualTo(200); |
||||
assertThat(builder1.level0SlowdownWritesTrigger()).isEqualTo(312); |
||||
assertThat(builder1.level0StopWritesTrigger()).isEqualTo(584); |
||||
assertThat(builder1.maxCompactionBytes()).isEqualTo(12); |
||||
assertThat(builder1.targetFileSizeBase()).isEqualTo(99); |
||||
assertThat(builder1.targetFileSizeMultiplier()).isEqualTo(112); |
||||
|
||||
final MutableColumnFamilyOptions |
||||
.MutableColumnFamilyOptionsBuilder mutableColumnFamilyOptions2 = |
||||
MutableColumnFamilyOptions.builder() |
||||
.setMinBlobSize(minBlobSize) |
||||
.setEnableBlobFiles(false) |
||||
.setArenaBlockSize(42) |
||||
.setMemtablePrefixBloomSizeRatio(0.236) |
||||
.setMemtableHugePageSize(8) |
||||
.setMaxSuccessiveMerges(12) |
||||
.setMaxWriteBufferNumber(22) |
||||
.setInplaceUpdateNumLocks(160) |
||||
.setDisableAutoCompactions(true) |
||||
.setSoftPendingCompactionBytesLimit(1124) |
||||
.setHardPendingCompactionBytesLimit(2800) |
||||
.setLevel0FileNumCompactionTrigger(2000) |
||||
.setLevel0SlowdownWritesTrigger(5840) |
||||
.setLevel0StopWritesTrigger(31200) |
||||
.setMaxCompactionBytes(112) |
||||
.setTargetFileSizeBase(999) |
||||
.setTargetFileSizeMultiplier(1120); |
||||
db.setOptions(columnFamilyHandle2, mutableColumnFamilyOptions2.build()); |
||||
|
||||
// Check the getOptions() brings back the creation options for CF2
|
||||
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder2 = |
||||
db.getOptions(columnFamilyHandle2); |
||||
assertThat(builder2.enableBlobFiles()).isEqualTo(false); |
||||
assertThat(builder2.minBlobSize()).isEqualTo(minBlobSize); |
||||
assertThat(builder2.arenaBlockSize()).isEqualTo(42); |
||||
assertThat(builder2.memtableHugePageSize()).isEqualTo(8); |
||||
assertThat(builder2.memtablePrefixBloomSizeRatio()).isEqualTo(0.236); |
||||
assertThat(builder2.maxSuccessiveMerges()).isEqualTo(12); |
||||
assertThat(builder2.maxWriteBufferNumber()).isEqualTo(22); |
||||
assertThat(builder2.inplaceUpdateNumLocks()).isEqualTo(160); |
||||
assertThat(builder2.disableAutoCompactions()).isEqualTo(true); |
||||
assertThat(builder2.softPendingCompactionBytesLimit()).isEqualTo(1124); |
||||
assertThat(builder2.hardPendingCompactionBytesLimit()).isEqualTo(2800); |
||||
assertThat(builder2.level0FileNumCompactionTrigger()).isEqualTo(2000); |
||||
assertThat(builder2.level0SlowdownWritesTrigger()).isEqualTo(5840); |
||||
assertThat(builder2.level0StopWritesTrigger()).isEqualTo(31200); |
||||
assertThat(builder2.maxCompactionBytes()).isEqualTo(112); |
||||
assertThat(builder2.targetFileSizeBase()).isEqualTo(999); |
||||
assertThat(builder2.targetFileSizeMultiplier()).isEqualTo(1120); |
||||
} |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Validate the round-trip of blob options into and out of the C++ core of RocksDB |
||||
* From {RocksDB#setOptions} to {RocksDB#getOptions} |
||||
* Uses 2x column families with different values for their options. |
||||
* NOTE that some constraints are applied to the options in the C++ core, |
||||
* e.g. on {ColumnFamilyOptions#setMemtablePrefixBloomSizeRatio} |
||||
* |
||||
* @throws RocksDBException if a database access has an error |
||||
*/ |
||||
@Test |
||||
public void testGetMutableBlobOptionsAfterSet() throws RocksDBException { |
||||
final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor0 = |
||||
new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); |
||||
final List<ColumnFamilyDescriptor> columnFamilyDescriptors = |
||||
Collections.singletonList(columnFamilyDescriptor0); |
||||
final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(); |
||||
|
||||
try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); |
||||
final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), |
||||
columnFamilyDescriptors, columnFamilyHandles)) { |
||||
final MutableColumnFamilyOptions |
||||
.MutableColumnFamilyOptionsBuilder mutableColumnFamilyOptions = |
||||
MutableColumnFamilyOptions.builder() |
||||
.setMinBlobSize(minBlobSize) |
||||
.setEnableBlobFiles(true) |
||||
.setArenaBlockSize(42) |
||||
.setMemtablePrefixBloomSizeRatio(0.17) |
||||
.setMemtableHugePageSize(3) |
||||
.setMaxSuccessiveMerges(4) |
||||
.setMaxWriteBufferNumber(12) |
||||
.setInplaceUpdateNumLocks(16) |
||||
.setDisableAutoCompactions(false) |
||||
.setSoftPendingCompactionBytesLimit(112) |
||||
.setHardPendingCompactionBytesLimit(280) |
||||
.setLevel0FileNumCompactionTrigger(200) |
||||
.setLevel0SlowdownWritesTrigger(312) |
||||
.setLevel0StopWritesTrigger(584) |
||||
.setMaxCompactionBytes(12) |
||||
.setTargetFileSizeBase(99) |
||||
.setTargetFileSizeMultiplier(112); |
||||
db.setOptions(mutableColumnFamilyOptions.build()); |
||||
|
||||
// Check the getOptions() brings back the creation options for CF1
|
||||
final MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder builder1 = db.getOptions(); |
||||
assertThat(builder1.enableBlobFiles()).isEqualTo(true); |
||||
assertThat(builder1.minBlobSize()).isEqualTo(minBlobSize); |
||||
assertThat(builder1.arenaBlockSize()).isEqualTo(42); |
||||
assertThat(builder1.memtableHugePageSize()).isEqualTo(3); |
||||
assertThat(builder1.memtablePrefixBloomSizeRatio()).isEqualTo(0.17); |
||||
assertThat(builder1.maxSuccessiveMerges()).isEqualTo(4); |
||||
assertThat(builder1.maxWriteBufferNumber()).isEqualTo(12); |
||||
assertThat(builder1.inplaceUpdateNumLocks()).isEqualTo(16); |
||||
assertThat(builder1.disableAutoCompactions()).isEqualTo(false); |
||||
assertThat(builder1.softPendingCompactionBytesLimit()).isEqualTo(112); |
||||
assertThat(builder1.hardPendingCompactionBytesLimit()).isEqualTo(280); |
||||
assertThat(builder1.level0FileNumCompactionTrigger()).isEqualTo(200); |
||||
assertThat(builder1.level0SlowdownWritesTrigger()).isEqualTo(312); |
||||
assertThat(builder1.level0StopWritesTrigger()).isEqualTo(584); |
||||
assertThat(builder1.maxCompactionBytes()).isEqualTo(12); |
||||
assertThat(builder1.targetFileSizeBase()).isEqualTo(99); |
||||
assertThat(builder1.targetFileSizeMultiplier()).isEqualTo(112); |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
public void testGetMutableDBOptionsAfterSet() throws RocksDBException { |
||||
final ColumnFamilyOptions columnFamilyOptions0 = new ColumnFamilyOptions(); |
||||
final ColumnFamilyDescriptor columnFamilyDescriptor0 = |
||||
new ColumnFamilyDescriptor("default".getBytes(UTF_8), columnFamilyOptions0); |
||||
final List<ColumnFamilyDescriptor> columnFamilyDescriptors = |
||||
Collections.singletonList(columnFamilyDescriptor0); |
||||
final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(); |
||||
|
||||
try (final DBOptions dbOptions = new DBOptions().setCreateIfMissing(true); |
||||
final RocksDB db = RocksDB.open(dbOptions, dbFolder.getRoot().getAbsolutePath(), |
||||
columnFamilyDescriptors, columnFamilyHandles)) { |
||||
final MutableDBOptions.MutableDBOptionsBuilder mutableDBOptions = |
||||
MutableDBOptions.builder() |
||||
.setMaxBackgroundJobs(16) |
||||
.setAvoidFlushDuringShutdown(true) |
||||
.setWritableFileMaxBufferSize(2097152) |
||||
.setDelayedWriteRate(67108864) |
||||
.setMaxTotalWalSize(16777216) |
||||
.setDeleteObsoleteFilesPeriodMicros(86400000000L) |
||||
.setStatsDumpPeriodSec(1200) |
||||
.setStatsPersistPeriodSec(7200) |
||||
.setStatsHistoryBufferSize(6291456) |
||||
.setMaxOpenFiles(8) |
||||
.setBytesPerSync(4194304) |
||||
.setWalBytesPerSync(1048576) |
||||
.setStrictBytesPerSync(true) |
||||
.setCompactionReadaheadSize(1024); |
||||
|
||||
db.setDBOptions(mutableDBOptions.build()); |
||||
|
||||
final MutableDBOptions.MutableDBOptionsBuilder getBuilder = db.getDBOptions(); |
||||
assertThat(getBuilder.maxBackgroundJobs()).isEqualTo(16); // 4
|
||||
assertThat(getBuilder.avoidFlushDuringShutdown()).isEqualTo(true); // false
|
||||
assertThat(getBuilder.writableFileMaxBufferSize()).isEqualTo(2097152); // 1048576
|
||||
assertThat(getBuilder.delayedWriteRate()).isEqualTo(67108864); // 16777216
|
||||
assertThat(getBuilder.maxTotalWalSize()).isEqualTo(16777216); |
||||
assertThat(getBuilder.deleteObsoleteFilesPeriodMicros()) |
||||
.isEqualTo(86400000000L); // 21600000000
|
||||
assertThat(getBuilder.statsDumpPeriodSec()).isEqualTo(1200); // 600
|
||||
assertThat(getBuilder.statsPersistPeriodSec()).isEqualTo(7200); // 600
|
||||
assertThat(getBuilder.statsHistoryBufferSize()).isEqualTo(6291456); // 1048576
|
||||
assertThat(getBuilder.maxOpenFiles()).isEqualTo(8); //-1
|
||||
assertThat(getBuilder.bytesPerSync()).isEqualTo(4194304); // 1048576
|
||||
assertThat(getBuilder.walBytesPerSync()).isEqualTo(1048576); // 0
|
||||
assertThat(getBuilder.strictBytesPerSync()).isEqualTo(true); // false
|
||||
assertThat(getBuilder.compactionReadaheadSize()).isEqualTo(1024); // 0
|
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,79 @@ |
||||
# How RocksDB Options and their Java Wrappers Work |
||||
|
||||
Options in RocksDB come in many different flavours. This is an attempt at a taxonomy and explanation. |
||||
|
||||
## RocksDB Options |
||||
|
||||
Initially, I believe, RocksDB had only database options. I don't know if any of these were mutable. Column families came later. Read on to understand the terminology. |
||||
|
||||
So to begin, one sets up a collection of options and starts/creates a database with these options. That's a useful way to think about it, because from a Java point-of-view (and I didn't realise this initially and got very confused), despite making native calls to C++, the `API`s are just manipulating a native C++ configuration object. This object is just a record of configuration, and it must later be passed to the database (at create or open time) in order to apply the options. |
||||
|
||||
### Database versus Column Family |
||||
|
||||
The concept of the *column family* or `CF` is widespread within RocksDB. I think of it as a data namespace, but conveniently transactions can operate across these namespaces. The concept of a default column family exists, and when operations do not refer to a particular `CF`, it refers to the default. |
||||
|
||||
We raise this w.r.t. options because many options, perhaps most that users encounter, are *column family options*. That is to say they apply individually to a particular column family, or to the default column family. Crucially also, many/most/all of these same options are exposed as *database options* and then apply as the default for column families which do not have the option set explicitly. Obviously some database options are naturally database-wide; they apply to the operation of the database and don't make any sense applied to a column family. |
||||
|
||||
### Mutability |
||||
|
||||
There are 2 kinds of options |
||||
|
||||
- Mutable options |
||||
- Immutable options. We name these in contrast to the mutable ones, but they are usually referred to unqualified. |
||||
|
||||
Mutable options are those which can be changed on a running `RocksDB` instance. Immutable options can only be configured prior to the start of a database. Of course, we can configure the immutable options at this time too; The entirety of options is a strict superset of the mutable options. |
||||
|
||||
Mutable options (whether column-family specific or database-wide) are manipulated at runtime with builders, so we have `MutableDBOptions.MutableDBOptionsBuilder` and `MutableColumnFamilyOptions.MutableColumnFamilyOptionsBuilder` which share tooling classes/hierarchy and maintain and manipulate the relevant options as a `(key,value)` map. |
||||
|
||||
Mutable options are then passed using `setOptions()` and `setDBOptions()` methods on the live RocksDB, and then take effect immediately (depending on the semantics of the option) on the database. |
||||
|
||||
### Advanced |
||||
|
||||
There are 2 classes of options |
||||
|
||||
- Advanced options |
||||
- Non-advanced options |
||||
|
||||
It's not clear to me what the conceptual distinction is between advanced and not. However, the Java code takes care to reflect it from the underlying C++. |
||||
|
||||
This leads to 2 separate type hierarchies within column family options, one for each `class` of options. The `kind`s are represented by where the options appear in their hierarchy. |
||||
|
||||
```java |
||||
interface ColumnFamilyOptionsInterface<T extends ColumnFamilyOptionsInterface<T>> |
||||
extends AdvancedColumnFamilyOptionsInterface<T> |
||||
interface MutableColumnFamilyOptionsInterface<T extends MutableColumnFamilyOptionsInterface<T>> |
||||
extends AdvancedMutableColumnFamilyOptionsInterface<T> |
||||
``` |
||||
|
||||
And then there is ultimately a single concrete implementation class for CF options: |
||||
|
||||
```java |
||||
class ColumnFamilyOptions extends RocksObject |
||||
implements ColumnFamilyOptionsInterface<ColumnFamilyOptions>, |
||||
MutableColumnFamilyOptionsInterface<ColumnFamilyOptions> |
||||
``` |
||||
|
||||
as there is a single concrete implementation class for DB options: |
||||
|
||||
```java |
||||
class DBOptions extends RocksObject |
||||
implements DBOptionsInterface<DBOptions>, |
||||
MutableDBOptionsInterface<DBOptions> |
||||
``` |
||||
|
||||
Interestingly `DBOptionsInterface` doesn't extend `MutableDBOptionsInterface`, if only in order to disrupt our belief in consistent basic laws of the Universe. |
||||
|
||||
## Startup/Creation Options |
||||
|
||||
```java |
||||
class Options extends RocksObject |
||||
implements DBOptionsInterface<Options>, |
||||
MutableDBOptionsInterface<Options>, |
||||
ColumnFamilyOptionsInterface<Options>, |
||||
MutableColumnFamilyOptionsInterface<Options> |
||||
``` |
||||
|
||||
### Example - Blob Options |
||||
|
||||
The `enable_blob_files` and `min_blob_size` options are per-column-family, and are mutable. The options also appear in the unqualified database options. So by initial configuration, we can set up a RocksDB database where for every `(key,value)` with a value of size at least `min_blob_size`, the value is written (indirected) to a blob file. Blobs may share a blob file, subject to the configuration values set. Later, using the `MutableColumnFamilyOptionsInterface` of the `ColumnFamilyOptions`, we can choose to turn this off (`enable_blob_files=false`) , or alter the `min_blob_size` for the default column family, or any other column family. It seems to me that we cannot, though, mutate the column family options for all column families using the |
||||
`setOptions()` mechanism, either for all existing column families or for all future column families; but maybe we can do the latter on a re-`open()/create()' |
||||
Loading…
Reference in new issue