Fix Java API ComparatorOptions use after delete error (#11176)
Summary: The problem ------------- ComparatorOptions is AutoCloseable. AbstractComparator does not hold a reference to its ComparatorOptions, but the native C++ ComparatorJniCallback holds a reference to the ComparatorOptions’ native C++ options structure. This gets deleted when the ComparatorOptions is closed, either explicitly, or as part of try-with-resources. Later, the deleted C++ options structure gets used by the callback and the comparator options are effectively random. The original bug report https://github.com/facebook/rocksdb/issues/8715 was caused by a GC-initiated finalization closing the still-in-use ComparatorOptions. As of 7.0, finalization of RocksDB objects no longer closes them, which worked round the reported bug, but still left ComparatorOptions with a potentially broken lifetime. In any case, we encourage API clients to use the try-with-resources model, and so we need it to work. And if they don't use it, they leak resources. The solution ------------- The solution implemented here is to make a copy of the native C++ options object into the ComparatorJniCallback, rather than a reference. Then the deletion of the native object held by ComparatorOptions is *correctly* deleted when its scope is closed in try/finally. Testing ------- We added a regression unit test based on the original test for the reported ticket. This checkin closes https://github.com/facebook/rocksdb/issues/8715 We expect that there are more instances of "lifecycle" bugs in the Java API. They are a major source of support time/cost, and we note that they could be addressed as a whole using the model proposed/prototyped in https://github.com/facebook/rocksdb/pull/10736 Pull Request resolved: https://github.com/facebook/rocksdb/pull/11176 Reviewed By: cbi42 Differential Revision: D43160885 Pulled By: pdillinger fbshipit-source-id: 60b54215a02ad9abb17363319650328c00a9ad62oxigraph-8.1.1
Alan Paxton
2 years ago
committed by
Facebook GitHub Bot
parent
b6640c3117
commit
d47126875b
@ -0,0 +1,132 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb; |
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets; |
||||||
|
import java.util.*; |
||||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||||
|
import org.junit.ClassRule; |
||||||
|
import org.junit.Rule; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.rules.TemporaryFolder; |
||||||
|
import org.rocksdb.util.ReverseBytewiseComparator; |
||||||
|
|
||||||
|
public class ByteBufferUnsupportedOperationTest { |
||||||
|
@ClassRule |
||||||
|
public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = |
||||||
|
new RocksNativeLibraryResource(); |
||||||
|
|
||||||
|
@Rule public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||||
|
|
||||||
|
public static class Handler { |
||||||
|
private final RocksDB database; |
||||||
|
private final Map<UUID, ColumnFamilyHandle> columnFamilies; |
||||||
|
|
||||||
|
public Handler(final String path, final Options options) throws RocksDBException { |
||||||
|
RocksDB.destroyDB(path, options); |
||||||
|
this.database = RocksDB.open(options, path); |
||||||
|
this.columnFamilies = new ConcurrentHashMap<>(); |
||||||
|
} |
||||||
|
|
||||||
|
public void addTable(final UUID streamID) throws RocksDBException { |
||||||
|
final ColumnFamilyOptions tableOptions = new ColumnFamilyOptions(); |
||||||
|
tableOptions.optimizeUniversalStyleCompaction(); |
||||||
|
try (final ComparatorOptions comparatorOptions = new ComparatorOptions()) { |
||||||
|
// comparatorOptions.setReusedSynchronisationType(ReusedSynchronisationType.ADAPTIVE_MUTEX);
|
||||||
|
tableOptions.setComparator(new ReverseBytewiseComparator(comparatorOptions)); |
||||||
|
final ColumnFamilyDescriptor tableDescriptor = new ColumnFamilyDescriptor( |
||||||
|
streamID.toString().getBytes(StandardCharsets.UTF_8), tableOptions); |
||||||
|
final ColumnFamilyHandle tableHandle = database.createColumnFamily(tableDescriptor); |
||||||
|
columnFamilies.put(streamID, tableHandle); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public void updateAll(final List<byte[][]> keyValuePairs, final UUID streamID) |
||||||
|
throws RocksDBException { |
||||||
|
final ColumnFamilyHandle currTable = columnFamilies.get(streamID); |
||||||
|
try (final WriteBatch batchedWrite = new WriteBatch(); |
||||||
|
final WriteOptions writeOptions = new WriteOptions()) { |
||||||
|
for (final byte[][] pair : keyValuePairs) { |
||||||
|
final byte[] keyBytes = pair[0]; |
||||||
|
final byte[] valueBytes = pair[1]; |
||||||
|
batchedWrite.put(currTable, keyBytes, valueBytes); |
||||||
|
} |
||||||
|
database.write(writeOptions, batchedWrite); |
||||||
|
} |
||||||
|
} |
||||||
|
public boolean containsValue(final byte[] encodedValue, final UUID streamID) { |
||||||
|
try (final RocksIterator iter = database.newIterator(columnFamilies.get(streamID))) { |
||||||
|
iter.seekToFirst(); |
||||||
|
while (iter.isValid()) { |
||||||
|
final byte[] val = iter.value(); |
||||||
|
if (Arrays.equals(val, encodedValue)) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
iter.next(); |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
public void close() { |
||||||
|
for (final ColumnFamilyHandle handle : columnFamilies.values()) { |
||||||
|
handle.close(); |
||||||
|
} |
||||||
|
database.close(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private void inner(final int numRepeats) throws RocksDBException { |
||||||
|
final Options opts = new Options(); |
||||||
|
opts.setCreateIfMissing(true); |
||||||
|
final Handler handler = new Handler("testDB", opts); |
||||||
|
final UUID stream1 = UUID.randomUUID(); |
||||||
|
|
||||||
|
final List<byte[][]> entries = new ArrayList<>(); |
||||||
|
for (int i = 0; i < numRepeats; i++) { |
||||||
|
final byte[] value = value(i); |
||||||
|
final byte[] key = key(i); |
||||||
|
entries.add(new byte[][] {key, value}); |
||||||
|
} |
||||||
|
handler.addTable(stream1); |
||||||
|
handler.updateAll(entries, stream1); |
||||||
|
|
||||||
|
for (int i = 0; i < numRepeats; i++) { |
||||||
|
final byte[] val = value(i); |
||||||
|
final boolean hasValue = handler.containsValue(val, stream1); |
||||||
|
if (!hasValue) { |
||||||
|
throw new IllegalStateException("not has value " + i); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
handler.close(); |
||||||
|
} |
||||||
|
|
||||||
|
private static byte[] key(final int i) { |
||||||
|
return ("key" + i).getBytes(StandardCharsets.UTF_8); |
||||||
|
} |
||||||
|
|
||||||
|
private static byte[] value(final int i) { |
||||||
|
return ("value" + i).getBytes(StandardCharsets.UTF_8); |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void unsupportedOperation() throws RocksDBException { |
||||||
|
final int numRepeats = 1000; |
||||||
|
final int repeatTest = 10; |
||||||
|
|
||||||
|
// the error is not always reproducible... let's try to increase the odds by repeating the main
|
||||||
|
// test body
|
||||||
|
for (int i = 0; i < repeatTest; i++) { |
||||||
|
try { |
||||||
|
inner(numRepeats); |
||||||
|
} catch (final RuntimeException runtimeException) { |
||||||
|
System.out.println("Exception on repeat " + i); |
||||||
|
throw runtimeException; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue