Fix Java API ComparatorOptions use after delete error (#11176)
Summary: The problem ------------- ComparatorOptions is AutoCloseable. AbstractComparator does not hold a reference to its ComparatorOptions, but the native C++ ComparatorJniCallback holds a reference to the ComparatorOptions’ native C++ options structure. This gets deleted when the ComparatorOptions is closed, either explicitly, or as part of try-with-resources. Later, the deleted C++ options structure gets used by the callback and the comparator options are effectively random. The original bug report https://github.com/facebook/rocksdb/issues/8715 was caused by a GC-initiated finalization closing the still-in-use ComparatorOptions. As of 7.0, finalization of RocksDB objects no longer closes them, which worked round the reported bug, but still left ComparatorOptions with a potentially broken lifetime. In any case, we encourage API clients to use the try-with-resources model, and so we need it to work. And if they don't use it, they leak resources. The solution ------------- The solution implemented here is to make a copy of the native C++ options object into the ComparatorJniCallback, rather than a reference. Then the deletion of the native object held by ComparatorOptions is *correctly* deleted when its scope is closed in try/finally. Testing ------- We added a regression unit test based on the original test for the reported ticket. This checkin closes https://github.com/facebook/rocksdb/issues/8715 We expect that there are more instances of "lifecycle" bugs in the Java API. They are a major source of support time/cost, and we note that they could be addressed as a whole using the model proposed/prototyped in https://github.com/facebook/rocksdb/pull/10736 Pull Request resolved: https://github.com/facebook/rocksdb/pull/11176 Reviewed By: cbi42 Differential Revision: D43160885 Pulled By: pdillinger fbshipit-source-id: 60b54215a02ad9abb17363319650328c00a9ad62oxigraph-8.1.1
Alan Paxton
2 years ago
committed by
Facebook GitHub Bot
parent
b6640c3117
commit
d47126875b
@ -0,0 +1,132 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
package org.rocksdb; |
||||
|
||||
import java.nio.charset.StandardCharsets; |
||||
import java.util.*; |
||||
import java.util.concurrent.ConcurrentHashMap; |
||||
import org.junit.ClassRule; |
||||
import org.junit.Rule; |
||||
import org.junit.Test; |
||||
import org.junit.rules.TemporaryFolder; |
||||
import org.rocksdb.util.ReverseBytewiseComparator; |
||||
|
||||
public class ByteBufferUnsupportedOperationTest { |
||||
@ClassRule |
||||
public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = |
||||
new RocksNativeLibraryResource(); |
||||
|
||||
@Rule public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||
|
||||
public static class Handler { |
||||
private final RocksDB database; |
||||
private final Map<UUID, ColumnFamilyHandle> columnFamilies; |
||||
|
||||
public Handler(final String path, final Options options) throws RocksDBException { |
||||
RocksDB.destroyDB(path, options); |
||||
this.database = RocksDB.open(options, path); |
||||
this.columnFamilies = new ConcurrentHashMap<>(); |
||||
} |
||||
|
||||
public void addTable(final UUID streamID) throws RocksDBException { |
||||
final ColumnFamilyOptions tableOptions = new ColumnFamilyOptions(); |
||||
tableOptions.optimizeUniversalStyleCompaction(); |
||||
try (final ComparatorOptions comparatorOptions = new ComparatorOptions()) { |
||||
// comparatorOptions.setReusedSynchronisationType(ReusedSynchronisationType.ADAPTIVE_MUTEX);
|
||||
tableOptions.setComparator(new ReverseBytewiseComparator(comparatorOptions)); |
||||
final ColumnFamilyDescriptor tableDescriptor = new ColumnFamilyDescriptor( |
||||
streamID.toString().getBytes(StandardCharsets.UTF_8), tableOptions); |
||||
final ColumnFamilyHandle tableHandle = database.createColumnFamily(tableDescriptor); |
||||
columnFamilies.put(streamID, tableHandle); |
||||
} |
||||
} |
||||
|
||||
public void updateAll(final List<byte[][]> keyValuePairs, final UUID streamID) |
||||
throws RocksDBException { |
||||
final ColumnFamilyHandle currTable = columnFamilies.get(streamID); |
||||
try (final WriteBatch batchedWrite = new WriteBatch(); |
||||
final WriteOptions writeOptions = new WriteOptions()) { |
||||
for (final byte[][] pair : keyValuePairs) { |
||||
final byte[] keyBytes = pair[0]; |
||||
final byte[] valueBytes = pair[1]; |
||||
batchedWrite.put(currTable, keyBytes, valueBytes); |
||||
} |
||||
database.write(writeOptions, batchedWrite); |
||||
} |
||||
} |
||||
public boolean containsValue(final byte[] encodedValue, final UUID streamID) { |
||||
try (final RocksIterator iter = database.newIterator(columnFamilies.get(streamID))) { |
||||
iter.seekToFirst(); |
||||
while (iter.isValid()) { |
||||
final byte[] val = iter.value(); |
||||
if (Arrays.equals(val, encodedValue)) { |
||||
return true; |
||||
} |
||||
iter.next(); |
||||
} |
||||
} |
||||
return false; |
||||
} |
||||
|
||||
public void close() { |
||||
for (final ColumnFamilyHandle handle : columnFamilies.values()) { |
||||
handle.close(); |
||||
} |
||||
database.close(); |
||||
} |
||||
} |
||||
|
||||
private void inner(final int numRepeats) throws RocksDBException { |
||||
final Options opts = new Options(); |
||||
opts.setCreateIfMissing(true); |
||||
final Handler handler = new Handler("testDB", opts); |
||||
final UUID stream1 = UUID.randomUUID(); |
||||
|
||||
final List<byte[][]> entries = new ArrayList<>(); |
||||
for (int i = 0; i < numRepeats; i++) { |
||||
final byte[] value = value(i); |
||||
final byte[] key = key(i); |
||||
entries.add(new byte[][] {key, value}); |
||||
} |
||||
handler.addTable(stream1); |
||||
handler.updateAll(entries, stream1); |
||||
|
||||
for (int i = 0; i < numRepeats; i++) { |
||||
final byte[] val = value(i); |
||||
final boolean hasValue = handler.containsValue(val, stream1); |
||||
if (!hasValue) { |
||||
throw new IllegalStateException("not has value " + i); |
||||
} |
||||
} |
||||
|
||||
handler.close(); |
||||
} |
||||
|
||||
private static byte[] key(final int i) { |
||||
return ("key" + i).getBytes(StandardCharsets.UTF_8); |
||||
} |
||||
|
||||
private static byte[] value(final int i) { |
||||
return ("value" + i).getBytes(StandardCharsets.UTF_8); |
||||
} |
||||
|
||||
@Test |
||||
public void unsupportedOperation() throws RocksDBException { |
||||
final int numRepeats = 1000; |
||||
final int repeatTest = 10; |
||||
|
||||
// the error is not always reproducible... let's try to increase the odds by repeating the main
|
||||
// test body
|
||||
for (int i = 0; i < repeatTest; i++) { |
||||
try { |
||||
inner(numRepeats); |
||||
} catch (final RuntimeException runtimeException) { |
||||
System.out.println("Exception on repeat " + i); |
||||
throw runtimeException; |
||||
} |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue