Improve RocksJava Comparator (#6252)
Summary: This is a redesign of the API for RocksJava comparators with the aim of improving performance. It also simplifies the class hierarchy. **NOTE**: This breaks backwards compatibility for existing 3rd party Comparators implemented in Java... so we need to consider carefully which release branches this goes into. Previously when implementing a comparator in Java the developer had a choice of subclassing either `DirectComparator` or `Comparator` which would use direct and non-direct byte-buffers resepectively (via `DirectSlice` and `Slice`). In this redesign there we have eliminated the overhead of using the Java Slice classes, and just use `ByteBuffer`s. The `ComparatorOptions` supplied when constructing a Comparator allow you to choose between direct and non-direct byte buffers by setting `useDirect`. In addition, the `ComparatorOptions` now allow you to choose whether a ByteBuffer is reused over multiple comparator calls, by setting `maxReusedBufferSize > 0`. When buffers are reused, ComparatorOptions provides a choice of mutex type by setting `useAdaptiveMutex`. --- [JMH benchmarks previously indicated](https://github.com/facebook/rocksdb/pull/6241#issue-356398306) that the difference between C++ and Java for implementing a comparator was ~7x slowdown in Java. With these changes, when reusing buffers and guarding access to them via mutexes the slowdown is approximately the same. However, these changes offer a new facility to not reuse mutextes, which reduces the slowdown to ~5.5x in Java. We also offer a `thread_local` mechanism for reusing buffers, which reduces slowdown to ~5.2x in Java (closes https://github.com/facebook/rocksdb/pull/4425). These changes also form a good base for further optimisation work such as further JNI lookup caching, and JNI critical. --- These numbers were captured without jemalloc. With jemalloc, the performance improves for all tests, and the Java slowdown reduces to between 4.8x and 5.x. ``` ComparatorBenchmarks.put native_bytewise thrpt 25 124483.795 ± 2032.443 ops/s ComparatorBenchmarks.put native_reverse_bytewise thrpt 25 114414.536 ± 3486.156 ops/s ComparatorBenchmarks.put java_bytewise_non-direct_reused-64_adaptive-mutex thrpt 25 17228.250 ± 1288.546 ops/s ComparatorBenchmarks.put java_bytewise_non-direct_reused-64_non-adaptive-mutex thrpt 25 16035.865 ± 1248.099 ops/s ComparatorBenchmarks.put java_bytewise_non-direct_reused-64_thread-local thrpt 25 21571.500 ± 871.521 ops/s ComparatorBenchmarks.put java_bytewise_direct_reused-64_adaptive-mutex thrpt 25 23613.773 ± 8465.660 ops/s ComparatorBenchmarks.put java_bytewise_direct_reused-64_non-adaptive-mutex thrpt 25 16768.172 ± 5618.489 ops/s ComparatorBenchmarks.put java_bytewise_direct_reused-64_thread-local thrpt 25 23921.164 ± 8734.742 ops/s ComparatorBenchmarks.put java_bytewise_non-direct_no-reuse thrpt 25 17899.684 ± 839.679 ops/s ComparatorBenchmarks.put java_bytewise_direct_no-reuse thrpt 25 22148.316 ± 1215.527 ops/s ComparatorBenchmarks.put java_reverse_bytewise_non-direct_reused-64_adaptive-mutex thrpt 25 11311.126 ± 820.602 ops/s ComparatorBenchmarks.put java_reverse_bytewise_non-direct_reused-64_non-adaptive-mutex thrpt 25 11421.311 ± 807.210 ops/s ComparatorBenchmarks.put java_reverse_bytewise_non-direct_reused-64_thread-local thrpt 25 11554.005 ± 960.556 ops/s ComparatorBenchmarks.put java_reverse_bytewise_direct_reused-64_adaptive-mutex thrpt 25 22960.523 ± 1673.421 ops/s ComparatorBenchmarks.put java_reverse_bytewise_direct_reused-64_non-adaptive-mutex thrpt 25 18293.317 ± 1434.601 ops/s ComparatorBenchmarks.put java_reverse_bytewise_direct_reused-64_thread-local thrpt 25 24479.361 ± 2157.306 ops/s ComparatorBenchmarks.put java_reverse_bytewise_non-direct_no-reuse thrpt 25 7942.286 ± 626.170 ops/s ComparatorBenchmarks.put java_reverse_bytewise_direct_no-reuse thrpt 25 11781.955 ± 1019.843 ops/s ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/6252 Differential Revision: D19331064 Pulled By: pdillinger fbshipit-source-id: 1f3b794e6a14162b2c3ffb943e8c0e64a0c03738main
Adam Retter
5 years ago
committed by
Facebook Github Bot
parent
800d24ddc5
commit
7242dae7fe
@ -0,0 +1,125 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb; |
||||||
|
|
||||||
|
import java.nio.ByteBuffer; |
||||||
|
|
||||||
|
/** |
||||||
|
* This class is intentionally private, |
||||||
|
* it holds methods which are called |
||||||
|
* from C++ to interact with a Comparator |
||||||
|
* written in Java. |
||||||
|
* |
||||||
|
* Placing these bridge methods in this |
||||||
|
* class keeps the API of the |
||||||
|
* {@link org.rocksdb.AbstractComparator} clean. |
||||||
|
*/ |
||||||
|
class AbstractComparatorJniBridge { |
||||||
|
|
||||||
|
/** |
||||||
|
* Only called from JNI. |
||||||
|
* |
||||||
|
* Simply a bridge to calling |
||||||
|
* {@link AbstractComparator#compare(ByteBuffer, ByteBuffer)}, |
||||||
|
* which ensures that the byte buffer lengths are correct |
||||||
|
* before and after the call. |
||||||
|
* |
||||||
|
* @param comparator the comparator object on which to |
||||||
|
* call {@link AbstractComparator#compare(ByteBuffer, ByteBuffer)} |
||||||
|
* @param a buffer access to first key |
||||||
|
* @param aLen the length of the a key, |
||||||
|
* may be smaller than the buffer {@code a} |
||||||
|
* @param b buffer access to second key |
||||||
|
* @param bLen the length of the b key, |
||||||
|
* may be smaller than the buffer {@code b} |
||||||
|
* |
||||||
|
* @return the result of the comparison |
||||||
|
*/ |
||||||
|
private static int compareInternal( |
||||||
|
final AbstractComparator comparator, |
||||||
|
final ByteBuffer a, final int aLen, |
||||||
|
final ByteBuffer b, final int bLen) { |
||||||
|
if (aLen != -1) { |
||||||
|
a.mark(); |
||||||
|
a.limit(aLen); |
||||||
|
} |
||||||
|
if (bLen != -1) { |
||||||
|
b.mark(); |
||||||
|
b.limit(bLen); |
||||||
|
} |
||||||
|
|
||||||
|
final int c = comparator.compare(a, b); |
||||||
|
|
||||||
|
if (aLen != -1) { |
||||||
|
a.reset(); |
||||||
|
} |
||||||
|
if (bLen != -1) { |
||||||
|
b.reset(); |
||||||
|
} |
||||||
|
|
||||||
|
return c; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Only called from JNI. |
||||||
|
* |
||||||
|
* Simply a bridge to calling |
||||||
|
* {@link AbstractComparator#findShortestSeparator(ByteBuffer, ByteBuffer)}, |
||||||
|
* which ensures that the byte buffer lengths are correct |
||||||
|
* before the call. |
||||||
|
* |
||||||
|
* @param comparator the comparator object on which to |
||||||
|
* call {@link AbstractComparator#findShortestSeparator(ByteBuffer, ByteBuffer)} |
||||||
|
* @param start buffer access to the start key |
||||||
|
* @param startLen the length of the start key, |
||||||
|
* may be smaller than the buffer {@code start} |
||||||
|
* @param limit buffer access to the limit key |
||||||
|
* @param limitLen the length of the limit key, |
||||||
|
* may be smaller than the buffer {@code limit} |
||||||
|
* |
||||||
|
* @return either {@code startLen} if the start key is unchanged, otherwise |
||||||
|
* the new length of the start key |
||||||
|
*/ |
||||||
|
private static int findShortestSeparatorInternal( |
||||||
|
final AbstractComparator comparator, |
||||||
|
final ByteBuffer start, final int startLen, |
||||||
|
final ByteBuffer limit, final int limitLen) { |
||||||
|
if (startLen != -1) { |
||||||
|
start.limit(startLen); |
||||||
|
} |
||||||
|
if (limitLen != -1) { |
||||||
|
limit.limit(limitLen); |
||||||
|
} |
||||||
|
comparator.findShortestSeparator(start, limit); |
||||||
|
return start.remaining(); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Only called from JNI. |
||||||
|
* |
||||||
|
* Simply a bridge to calling |
||||||
|
* {@link AbstractComparator#findShortestSeparator(ByteBuffer, ByteBuffer)}, |
||||||
|
* which ensures that the byte buffer length is correct |
||||||
|
* before the call. |
||||||
|
* |
||||||
|
* @param comparator the comparator object on which to |
||||||
|
* call {@link AbstractComparator#findShortSuccessor(ByteBuffer)} |
||||||
|
* @param key buffer access to the key |
||||||
|
* @param keyLen the length of the key, |
||||||
|
* may be smaller than the buffer {@code key} |
||||||
|
* |
||||||
|
* @return either keyLen if the key is unchanged, otherwise the new length of the key |
||||||
|
*/ |
||||||
|
private static int findShortSuccessorInternal( |
||||||
|
final AbstractComparator comparator, |
||||||
|
final ByteBuffer key, final int keyLen) { |
||||||
|
if (keyLen != -1) { |
||||||
|
key.limit(keyLen); |
||||||
|
} |
||||||
|
comparator.findShortSuccessor(key); |
||||||
|
return key.remaining(); |
||||||
|
} |
||||||
|
} |
||||||
@ -1,34 +0,0 @@ |
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
package org.rocksdb; |
|
||||||
|
|
||||||
/** |
|
||||||
* Base class for comparators which will receive |
|
||||||
* byte[] based access via org.rocksdb.Slice in their |
|
||||||
* compare method implementation. |
|
||||||
* |
|
||||||
* byte[] based slices perform better when small keys |
|
||||||
* are involved. When using larger keys consider |
|
||||||
* using @see org.rocksdb.DirectComparator |
|
||||||
*/ |
|
||||||
public abstract class Comparator extends AbstractComparator<Slice> { |
|
||||||
|
|
||||||
public Comparator(final ComparatorOptions copt) { |
|
||||||
super(copt); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
protected long initializeNative(final long... nativeParameterHandles) { |
|
||||||
return createNewComparator0(nativeParameterHandles[0]); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
final ComparatorType getComparatorType() { |
|
||||||
return ComparatorType.JAVA_COMPARATOR; |
|
||||||
} |
|
||||||
|
|
||||||
private native long createNewComparator0(final long comparatorOptionsHandle); |
|
||||||
} |
|
||||||
@ -1,35 +0,0 @@ |
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
package org.rocksdb; |
|
||||||
|
|
||||||
/** |
|
||||||
* Base class for comparators which will receive |
|
||||||
* ByteBuffer based access via org.rocksdb.DirectSlice |
|
||||||
* in their compare method implementation. |
|
||||||
* |
|
||||||
* ByteBuffer based slices perform better when large keys |
|
||||||
* are involved. When using smaller keys consider |
|
||||||
* using @see org.rocksdb.Comparator |
|
||||||
*/ |
|
||||||
public abstract class DirectComparator extends AbstractComparator<DirectSlice> { |
|
||||||
|
|
||||||
public DirectComparator(final ComparatorOptions copt) { |
|
||||||
super(copt); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
protected long initializeNative(final long... nativeParameterHandles) { |
|
||||||
return createNewDirectComparator0(nativeParameterHandles[0]); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
final ComparatorType getComparatorType() { |
|
||||||
return ComparatorType.JAVA_DIRECT_COMPARATOR; |
|
||||||
} |
|
||||||
|
|
||||||
private native long createNewDirectComparator0( |
|
||||||
final long comparatorOptionsHandle); |
|
||||||
} |
|
||||||
@ -0,0 +1,65 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
package org.rocksdb; |
||||||
|
|
||||||
|
/** |
||||||
|
* Determines the type of synchronisation primitive used |
||||||
|
* in native code. |
||||||
|
*/ |
||||||
|
public enum ReusedSynchronisationType { |
||||||
|
/** |
||||||
|
* Standard mutex. |
||||||
|
*/ |
||||||
|
MUTEX((byte)0x0), |
||||||
|
|
||||||
|
/** |
||||||
|
* Use adaptive mutex, which spins in the user space before resorting |
||||||
|
* to kernel. This could reduce context switch when the mutex is not |
||||||
|
* heavily contended. However, if the mutex is hot, we could end up |
||||||
|
* wasting spin time. |
||||||
|
*/ |
||||||
|
ADAPTIVE_MUTEX((byte)0x1), |
||||||
|
|
||||||
|
/** |
||||||
|
* There is a reused buffer per-thread. |
||||||
|
*/ |
||||||
|
THREAD_LOCAL((byte)0x2); |
||||||
|
|
||||||
|
private final byte value; |
||||||
|
|
||||||
|
ReusedSynchronisationType(final byte value) { |
||||||
|
this.value = value; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns the byte value of the enumerations value |
||||||
|
* |
||||||
|
* @return byte representation |
||||||
|
*/ |
||||||
|
public byte getValue() { |
||||||
|
return value; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Get ReusedSynchronisationType by byte value. |
||||||
|
* |
||||||
|
* @param value byte representation of ReusedSynchronisationType. |
||||||
|
* |
||||||
|
* @return {@link org.rocksdb.ReusedSynchronisationType} instance. |
||||||
|
* @throws java.lang.IllegalArgumentException if an invalid |
||||||
|
* value is provided. |
||||||
|
*/ |
||||||
|
public static ReusedSynchronisationType getReusedSynchronisationType( |
||||||
|
final byte value) { |
||||||
|
for (final ReusedSynchronisationType reusedSynchronisationType |
||||||
|
: ReusedSynchronisationType.values()) { |
||||||
|
if (reusedSynchronisationType.getValue() == value) { |
||||||
|
return reusedSynchronisationType; |
||||||
|
} |
||||||
|
} |
||||||
|
throw new IllegalArgumentException( |
||||||
|
"Illegal value provided for ReusedSynchronisationType."); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,46 @@ |
|||||||
|
package org.rocksdb.util; |
||||||
|
|
||||||
|
import java.nio.ByteBuffer; |
||||||
|
|
||||||
|
import static java.nio.charset.StandardCharsets.UTF_8; |
||||||
|
|
||||||
|
public class ByteUtil { |
||||||
|
|
||||||
|
/** |
||||||
|
* Convert a String to a UTF-8 byte array. |
||||||
|
* |
||||||
|
* @param str the string |
||||||
|
* |
||||||
|
* @return the byte array. |
||||||
|
*/ |
||||||
|
public static byte[] bytes(final String str) { |
||||||
|
return str.getBytes(UTF_8); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Compares the first {@code count} bytes of two areas of memory. Returns |
||||||
|
* zero if they are the same, a value less than zero if {@code x} is |
||||||
|
* lexically less than {@code y}, or a value greater than zero if {@code x} |
||||||
|
* is lexically greater than {@code y}. Note that lexical order is determined |
||||||
|
* as if comparing unsigned char arrays. |
||||||
|
* |
||||||
|
* Similar to <a href="https://github.com/gcc-mirror/gcc/blob/master/libiberty/memcmp.c">memcmp.c</a>. |
||||||
|
* |
||||||
|
* @param x the first value to compare with |
||||||
|
* @param y the second value to compare against |
||||||
|
* @param count the number of bytes to compare |
||||||
|
* |
||||||
|
* @return the result of the comparison |
||||||
|
*/ |
||||||
|
public static int memcmp(final ByteBuffer x, final ByteBuffer y, |
||||||
|
final int count) { |
||||||
|
for (int idx = 0; idx < count; idx++) { |
||||||
|
final int aa = x.get(idx) & 0xff; |
||||||
|
final int bb = y.get(idx) & 0xff; |
||||||
|
if (aa != bb) { |
||||||
|
return aa - bb; |
||||||
|
} |
||||||
|
} |
||||||
|
return 0; |
||||||
|
} |
||||||
|
} |
||||||
@ -1,88 +0,0 @@ |
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
package org.rocksdb.util; |
|
||||||
|
|
||||||
import org.rocksdb.ComparatorOptions; |
|
||||||
import org.rocksdb.DirectComparator; |
|
||||||
import org.rocksdb.DirectSlice; |
|
||||||
|
|
||||||
import java.nio.ByteBuffer; |
|
||||||
|
|
||||||
/** |
|
||||||
* This is a Java Native implementation of the C++ |
|
||||||
* equivalent BytewiseComparatorImpl using {@link DirectSlice} |
|
||||||
* |
|
||||||
* The performance of Comparators implemented in Java is always |
|
||||||
* less than their C++ counterparts due to the bridging overhead, |
|
||||||
* as such you likely don't want to use this apart from benchmarking |
|
||||||
* and you most likely instead wanted |
|
||||||
* {@link org.rocksdb.BuiltinComparator#BYTEWISE_COMPARATOR} |
|
||||||
*/ |
|
||||||
public class DirectBytewiseComparator extends DirectComparator { |
|
||||||
|
|
||||||
public DirectBytewiseComparator(final ComparatorOptions copt) { |
|
||||||
super(copt); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String name() { |
|
||||||
return "rocksdb.java.DirectBytewiseComparator"; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public int compare(final DirectSlice a, final DirectSlice b) { |
|
||||||
return a.data().compareTo(b.data()); |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String findShortestSeparator(final String start, |
|
||||||
final DirectSlice limit) { |
|
||||||
final byte[] startBytes = start.getBytes(); |
|
||||||
|
|
||||||
// Find length of common prefix
|
|
||||||
final int min_length = Math.min(startBytes.length, limit.size()); |
|
||||||
int diff_index = 0; |
|
||||||
while ((diff_index < min_length) && |
|
||||||
(startBytes[diff_index] == limit.get(diff_index))) { |
|
||||||
diff_index++; |
|
||||||
} |
|
||||||
|
|
||||||
if (diff_index >= min_length) { |
|
||||||
// Do not shorten if one string is a prefix of the other
|
|
||||||
} else { |
|
||||||
final byte diff_byte = startBytes[diff_index]; |
|
||||||
if(diff_byte < 0xff && diff_byte + 1 < limit.get(diff_index)) { |
|
||||||
final byte shortest[] = new byte[diff_index + 1]; |
|
||||||
System.arraycopy(startBytes, 0, shortest, 0, diff_index + 1); |
|
||||||
shortest[diff_index]++; |
|
||||||
assert(ByteBuffer.wrap(shortest).compareTo(limit.data()) < 0); |
|
||||||
return new String(shortest); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return null; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public String findShortSuccessor(final String key) { |
|
||||||
final byte[] keyBytes = key.getBytes(); |
|
||||||
|
|
||||||
// Find first character that can be incremented
|
|
||||||
final int n = keyBytes.length; |
|
||||||
for (int i = 0; i < n; i++) { |
|
||||||
final byte byt = keyBytes[i]; |
|
||||||
if (byt != 0xff) { |
|
||||||
final byte shortSuccessor[] = new byte[i + 1]; |
|
||||||
System.arraycopy(keyBytes, 0, shortSuccessor, 0, i + 1); |
|
||||||
shortSuccessor[i]++; |
|
||||||
return new String(shortSuccessor); |
|
||||||
} |
|
||||||
} |
|
||||||
// *key is a run of 0xffs. Leave it alone.
|
|
||||||
|
|
||||||
return null; |
|
||||||
} |
|
||||||
} |
|
||||||
@ -0,0 +1,67 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb.util; |
||||||
|
|
||||||
|
import org.rocksdb.AbstractComparator; |
||||||
|
import org.rocksdb.ComparatorOptions; |
||||||
|
|
||||||
|
import java.nio.ByteBuffer; |
||||||
|
|
||||||
|
/** |
||||||
|
* This is a Java implementation of a Comparator for Java int |
||||||
|
* keys. |
||||||
|
* |
||||||
|
* This comparator assumes keys are (at least) four bytes, so |
||||||
|
* the caller must guarantee that in accessing other APIs in |
||||||
|
* combination with this comparator. |
||||||
|
* |
||||||
|
* The performance of Comparators implemented in Java is always |
||||||
|
* less than their C++ counterparts due to the bridging overhead, |
||||||
|
* as such you likely don't want to use this apart from benchmarking |
||||||
|
* or testing. |
||||||
|
*/ |
||||||
|
public final class IntComparator extends AbstractComparator { |
||||||
|
|
||||||
|
public IntComparator(final ComparatorOptions copt) { |
||||||
|
super(copt); |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String name() { |
||||||
|
return "rocksdb.java.IntComparator"; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public int compare(final ByteBuffer a, final ByteBuffer b) { |
||||||
|
return compareIntKeys(a, b); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Compares integer keys |
||||||
|
* so that they are in ascending order |
||||||
|
* |
||||||
|
* @param a 4-bytes representing an integer key |
||||||
|
* @param b 4-bytes representing an integer key |
||||||
|
* |
||||||
|
* @return negative if a < b, 0 if a == b, positive otherwise |
||||||
|
*/ |
||||||
|
private final int compareIntKeys(final ByteBuffer a, final ByteBuffer b) { |
||||||
|
final int iA = a.getInt(); |
||||||
|
final int iB = b.getInt(); |
||||||
|
|
||||||
|
// protect against int key calculation overflow
|
||||||
|
final long diff = (long)iA - iB; |
||||||
|
final int result; |
||||||
|
if (diff < Integer.MIN_VALUE) { |
||||||
|
result = Integer.MIN_VALUE; |
||||||
|
} else if(diff > Integer.MAX_VALUE) { |
||||||
|
result = Integer.MAX_VALUE; |
||||||
|
} else { |
||||||
|
result = (int)diff; |
||||||
|
} |
||||||
|
return result; |
||||||
|
} |
||||||
|
} |
||||||
@ -1,199 +0,0 @@ |
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
package org.rocksdb; |
|
||||||
|
|
||||||
import java.io.IOException; |
|
||||||
import java.nio.file.*; |
|
||||||
import java.util.ArrayList; |
|
||||||
import java.util.Arrays; |
|
||||||
import java.util.List; |
|
||||||
import java.util.Random; |
|
||||||
|
|
||||||
import static org.assertj.core.api.Assertions.assertThat; |
|
||||||
import static org.rocksdb.Types.byteToInt; |
|
||||||
import static org.rocksdb.Types.intToByte; |
|
||||||
|
|
||||||
/** |
|
||||||
* Abstract tests for both Comparator and DirectComparator |
|
||||||
*/ |
|
||||||
public abstract class AbstractComparatorTest<T extends AbstractSlice<?>> { |
|
||||||
|
|
||||||
/** |
|
||||||
* Get a comparator which will expect Integer keys |
|
||||||
* and determine an ascending order |
|
||||||
* |
|
||||||
* @return An integer ascending order key comparator |
|
||||||
*/ |
|
||||||
public abstract AbstractComparator<T> getAscendingIntKeyComparator(); |
|
||||||
|
|
||||||
/** |
|
||||||
* Test which stores random keys into the database |
|
||||||
* using an @see getAscendingIntKeyComparator |
|
||||||
* it then checks that these keys are read back in |
|
||||||
* ascending order |
|
||||||
* |
|
||||||
* @param db_path A path where we can store database |
|
||||||
* files temporarily |
|
||||||
* |
|
||||||
* @throws java.io.IOException if IO error happens. |
|
||||||
*/ |
|
||||||
public void testRoundtrip(final Path db_path) throws IOException, |
|
||||||
RocksDBException { |
|
||||||
try (final AbstractComparator<T> comparator = getAscendingIntKeyComparator(); |
|
||||||
final Options opt = new Options() |
|
||||||
.setCreateIfMissing(true) |
|
||||||
.setComparator(comparator)) { |
|
||||||
|
|
||||||
// store 10,000 random integer keys
|
|
||||||
final int ITERATIONS = 10000; |
|
||||||
try (final RocksDB db = RocksDB.open(opt, db_path.toString())) { |
|
||||||
final Random random = new Random(); |
|
||||||
for (int i = 0; i < ITERATIONS; i++) { |
|
||||||
final byte[] key = intToByte(random.nextInt()); |
|
||||||
// does key already exist (avoid duplicates)
|
|
||||||
if (i > 0 && db.get(key) != null) { |
|
||||||
i--; // generate a different key
|
|
||||||
} else { |
|
||||||
db.put(key, "value".getBytes()); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// re-open db and read from start to end
|
|
||||||
// integer keys should be in ascending
|
|
||||||
// order as defined by SimpleIntComparator
|
|
||||||
try (final RocksDB db = RocksDB.open(opt, db_path.toString()); |
|
||||||
final RocksIterator it = db.newIterator()) { |
|
||||||
it.seekToFirst(); |
|
||||||
int lastKey = Integer.MIN_VALUE; |
|
||||||
int count = 0; |
|
||||||
for (it.seekToFirst(); it.isValid(); it.next()) { |
|
||||||
final int thisKey = byteToInt(it.key()); |
|
||||||
assertThat(thisKey).isGreaterThan(lastKey); |
|
||||||
lastKey = thisKey; |
|
||||||
count++; |
|
||||||
} |
|
||||||
assertThat(count).isEqualTo(ITERATIONS); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Test which stores random keys into a column family |
|
||||||
* in the database |
|
||||||
* using an @see getAscendingIntKeyComparator |
|
||||||
* it then checks that these keys are read back in |
|
||||||
* ascending order |
|
||||||
* |
|
||||||
* @param db_path A path where we can store database |
|
||||||
* files temporarily |
|
||||||
* |
|
||||||
* @throws java.io.IOException if IO error happens. |
|
||||||
*/ |
|
||||||
public void testRoundtripCf(final Path db_path) throws IOException, |
|
||||||
RocksDBException { |
|
||||||
|
|
||||||
try(final AbstractComparator<T> comparator = getAscendingIntKeyComparator()) { |
|
||||||
final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList( |
|
||||||
new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), |
|
||||||
new ColumnFamilyDescriptor("new_cf".getBytes(), |
|
||||||
new ColumnFamilyOptions().setComparator(comparator)) |
|
||||||
); |
|
||||||
|
|
||||||
final List<ColumnFamilyHandle> cfHandles = new ArrayList<>(); |
|
||||||
|
|
||||||
try (final DBOptions opt = new DBOptions(). |
|
||||||
setCreateIfMissing(true). |
|
||||||
setCreateMissingColumnFamilies(true)) { |
|
||||||
|
|
||||||
// store 10,000 random integer keys
|
|
||||||
final int ITERATIONS = 10000; |
|
||||||
|
|
||||||
try (final RocksDB db = RocksDB.open(opt, db_path.toString(), |
|
||||||
cfDescriptors, cfHandles)) { |
|
||||||
try { |
|
||||||
assertThat(cfDescriptors.size()).isEqualTo(2); |
|
||||||
assertThat(cfHandles.size()).isEqualTo(2); |
|
||||||
|
|
||||||
final Random random = new Random(); |
|
||||||
for (int i = 0; i < ITERATIONS; i++) { |
|
||||||
final byte key[] = intToByte(random.nextInt()); |
|
||||||
if (i > 0 && db.get(cfHandles.get(1), key) != null) { |
|
||||||
// does key already exist (avoid duplicates)
|
|
||||||
i--; // generate a different key
|
|
||||||
} else { |
|
||||||
db.put(cfHandles.get(1), key, "value".getBytes()); |
|
||||||
} |
|
||||||
} |
|
||||||
} finally { |
|
||||||
for (final ColumnFamilyHandle handle : cfHandles) { |
|
||||||
handle.close(); |
|
||||||
} |
|
||||||
} |
|
||||||
cfHandles.clear(); |
|
||||||
} |
|
||||||
|
|
||||||
// re-open db and read from start to end
|
|
||||||
// integer keys should be in ascending
|
|
||||||
// order as defined by SimpleIntComparator
|
|
||||||
try (final RocksDB db = RocksDB.open(opt, db_path.toString(), |
|
||||||
cfDescriptors, cfHandles); |
|
||||||
final RocksIterator it = db.newIterator(cfHandles.get(1))) { |
|
||||||
try { |
|
||||||
assertThat(cfDescriptors.size()).isEqualTo(2); |
|
||||||
assertThat(cfHandles.size()).isEqualTo(2); |
|
||||||
|
|
||||||
it.seekToFirst(); |
|
||||||
int lastKey = Integer.MIN_VALUE; |
|
||||||
int count = 0; |
|
||||||
for (it.seekToFirst(); it.isValid(); it.next()) { |
|
||||||
final int thisKey = byteToInt(it.key()); |
|
||||||
assertThat(thisKey).isGreaterThan(lastKey); |
|
||||||
lastKey = thisKey; |
|
||||||
count++; |
|
||||||
} |
|
||||||
|
|
||||||
assertThat(count).isEqualTo(ITERATIONS); |
|
||||||
|
|
||||||
} finally { |
|
||||||
for (final ColumnFamilyHandle handle : cfHandles) { |
|
||||||
handle.close(); |
|
||||||
} |
|
||||||
} |
|
||||||
cfHandles.clear(); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
/** |
|
||||||
* Compares integer keys |
|
||||||
* so that they are in ascending order |
|
||||||
* |
|
||||||
* @param a 4-bytes representing an integer key |
|
||||||
* @param b 4-bytes representing an integer key |
|
||||||
* |
|
||||||
* @return negative if a < b, 0 if a == b, positive otherwise |
|
||||||
*/ |
|
||||||
protected final int compareIntKeys(final byte[] a, final byte[] b) { |
|
||||||
|
|
||||||
final int iA = byteToInt(a); |
|
||||||
final int iB = byteToInt(b); |
|
||||||
|
|
||||||
// protect against int key calculation overflow
|
|
||||||
final double diff = (double)iA - iB; |
|
||||||
final int result; |
|
||||||
if (diff < Integer.MIN_VALUE) { |
|
||||||
result = Integer.MIN_VALUE; |
|
||||||
} else if(diff > Integer.MAX_VALUE) { |
|
||||||
result = Integer.MAX_VALUE; |
|
||||||
} else { |
|
||||||
result = (int)diff; |
|
||||||
} |
|
||||||
|
|
||||||
return result; |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,52 +0,0 @@ |
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
package org.rocksdb; |
|
||||||
|
|
||||||
import org.junit.ClassRule; |
|
||||||
import org.junit.Rule; |
|
||||||
import org.junit.Test; |
|
||||||
import org.junit.rules.TemporaryFolder; |
|
||||||
|
|
||||||
import java.io.IOException; |
|
||||||
import java.nio.file.FileSystems; |
|
||||||
|
|
||||||
public class DirectComparatorTest { |
|
||||||
@ClassRule |
|
||||||
public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = |
|
||||||
new RocksNativeLibraryResource(); |
|
||||||
|
|
||||||
@Rule |
|
||||||
public TemporaryFolder dbFolder = new TemporaryFolder(); |
|
||||||
|
|
||||||
@Test |
|
||||||
public void directComparator() throws IOException, RocksDBException { |
|
||||||
|
|
||||||
final AbstractComparatorTest<DirectSlice> comparatorTest = new AbstractComparatorTest<DirectSlice>() { |
|
||||||
@Override |
|
||||||
public AbstractComparator<DirectSlice> getAscendingIntKeyComparator() { |
|
||||||
return new DirectComparator(new ComparatorOptions()) { |
|
||||||
|
|
||||||
@Override |
|
||||||
public String name() { |
|
||||||
return "test.AscendingIntKeyDirectComparator"; |
|
||||||
} |
|
||||||
|
|
||||||
@Override |
|
||||||
public int compare(final DirectSlice a, final DirectSlice b) { |
|
||||||
final byte ax[] = new byte[4], bx[] = new byte[4]; |
|
||||||
a.data().get(ax); |
|
||||||
b.data().get(bx); |
|
||||||
return compareIntKeys(ax, bx); |
|
||||||
} |
|
||||||
}; |
|
||||||
} |
|
||||||
}; |
|
||||||
|
|
||||||
// test the round-tripability of keys written and read with the DirectComparator
|
|
||||||
comparatorTest.testRoundtrip(FileSystems.getDefault().getPath( |
|
||||||
dbFolder.getRoot().getAbsolutePath())); |
|
||||||
} |
|
||||||
} |
|
||||||
@ -0,0 +1,267 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb.util; |
||||||
|
|
||||||
|
import org.junit.BeforeClass; |
||||||
|
import org.junit.ClassRule; |
||||||
|
import org.junit.Rule; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.rules.TemporaryFolder; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.junit.runners.Parameterized; |
||||||
|
import org.junit.runners.Parameterized.Parameter; |
||||||
|
import org.junit.runners.Parameterized.Parameters; |
||||||
|
import org.rocksdb.*; |
||||||
|
|
||||||
|
import java.nio.ByteBuffer; |
||||||
|
import java.nio.file.FileSystems; |
||||||
|
import java.nio.file.Path; |
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Arrays; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Random; |
||||||
|
|
||||||
|
import static java.nio.charset.StandardCharsets.UTF_8; |
||||||
|
import static org.assertj.core.api.Assertions.assertThat; |
||||||
|
|
||||||
|
/** |
||||||
|
* Similar to {@link IntComparatorTest}, but uses {@link BytewiseComparator} |
||||||
|
* which ensures the correct ordering of positive integers. |
||||||
|
*/ |
||||||
|
@RunWith(Parameterized.class) |
||||||
|
public class BytewiseComparatorIntTest { |
||||||
|
|
||||||
|
// test with 500 random positive integer keys
|
||||||
|
private static final int TOTAL_KEYS = 500; |
||||||
|
private static final byte[][] keys = new byte[TOTAL_KEYS][4]; |
||||||
|
|
||||||
|
@BeforeClass |
||||||
|
public static void prepareKeys() { |
||||||
|
final ByteBuffer buf = ByteBuffer.allocate(4); |
||||||
|
final Random random = new Random(); |
||||||
|
for (int i = 0; i < TOTAL_KEYS; i++) { |
||||||
|
final int ri = random.nextInt() & Integer.MAX_VALUE; // the & ensures positive integer
|
||||||
|
buf.putInt(ri); |
||||||
|
buf.flip(); |
||||||
|
final byte[] key = buf.array(); |
||||||
|
|
||||||
|
// does key already exist (avoid duplicates)
|
||||||
|
if (keyExists(key, i)) { |
||||||
|
i--; // loop round and generate a different key
|
||||||
|
} else { |
||||||
|
System.arraycopy(key, 0, keys[i], 0, 4); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static boolean keyExists(final byte[] key, final int limit) { |
||||||
|
for (int j = 0; j < limit; j++) { |
||||||
|
if (Arrays.equals(key, keys[j])) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Parameters(name = "{0}") |
||||||
|
public static Iterable<Object[]> parameters() { |
||||||
|
return Arrays.asList(new Object[][] { |
||||||
|
{ "non-direct_reused64_mutex", false, 64, ReusedSynchronisationType.MUTEX }, |
||||||
|
{ "direct_reused64_mutex", true, 64, ReusedSynchronisationType.MUTEX }, |
||||||
|
{ "non-direct_reused64_adaptive-mutex", false, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, |
||||||
|
{ "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, |
||||||
|
{ "non-direct_reused64_thread-local", false, 64, ReusedSynchronisationType.THREAD_LOCAL }, |
||||||
|
{ "direct_reused64_thread-local", true, 64, ReusedSynchronisationType.THREAD_LOCAL }, |
||||||
|
{ "non-direct_noreuse", false, -1, null }, |
||||||
|
{ "direct_noreuse", true, -1, null } |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
@Parameter(0) |
||||||
|
public String name; |
||||||
|
|
||||||
|
@Parameter(1) |
||||||
|
public boolean useDirectBuffer; |
||||||
|
|
||||||
|
@Parameter(2) |
||||||
|
public int maxReusedBufferSize; |
||||||
|
|
||||||
|
@Parameter(3) |
||||||
|
public ReusedSynchronisationType reusedSynchronisationType; |
||||||
|
|
||||||
|
@ClassRule |
||||||
|
public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = |
||||||
|
new RocksNativeLibraryResource(); |
||||||
|
|
||||||
|
@Rule |
||||||
|
public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void javaComparatorDefaultCf() throws RocksDBException { |
||||||
|
try (final ComparatorOptions options = new ComparatorOptions() |
||||||
|
.setUseDirectBuffer(useDirectBuffer) |
||||||
|
.setMaxReusedBufferSize(maxReusedBufferSize) |
||||||
|
// if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used
|
||||||
|
.setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); |
||||||
|
final BytewiseComparator comparator = new BytewiseComparator(options)) { |
||||||
|
|
||||||
|
// test the round-tripability of keys written and read with the Comparator
|
||||||
|
testRoundtrip(FileSystems.getDefault().getPath( |
||||||
|
dbFolder.getRoot().getAbsolutePath()), comparator); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void javaComparatorNamedCf() throws RocksDBException { |
||||||
|
try (final ComparatorOptions options = new ComparatorOptions() |
||||||
|
.setUseDirectBuffer(useDirectBuffer) |
||||||
|
.setMaxReusedBufferSize(maxReusedBufferSize) |
||||||
|
// if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used
|
||||||
|
.setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); |
||||||
|
final BytewiseComparator comparator = new BytewiseComparator(options)) { |
||||||
|
|
||||||
|
// test the round-tripability of keys written and read with the Comparator
|
||||||
|
testRoundtripCf(FileSystems.getDefault().getPath( |
||||||
|
dbFolder.getRoot().getAbsolutePath()), comparator); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test which stores random keys into the database |
||||||
|
* using an {@link IntComparator} |
||||||
|
* it then checks that these keys are read back in |
||||||
|
* ascending order |
||||||
|
* |
||||||
|
* @param db_path A path where we can store database |
||||||
|
* files temporarily |
||||||
|
* |
||||||
|
* @param comparator the comparator |
||||||
|
* |
||||||
|
* @throws RocksDBException if a database error happens. |
||||||
|
*/ |
||||||
|
private void testRoundtrip(final Path db_path, |
||||||
|
final AbstractComparator comparator) throws RocksDBException { |
||||||
|
try (final Options opt = new Options() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setComparator(comparator)) { |
||||||
|
|
||||||
|
// store TOTAL_KEYS into the db
|
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString())) { |
||||||
|
for (int i = 0; i < TOTAL_KEYS; i++) { |
||||||
|
db.put(keys[i], "value".getBytes(UTF_8)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// re-open db and read from start to end
|
||||||
|
// integer keys should be in ascending
|
||||||
|
// order as defined by IntComparator
|
||||||
|
final ByteBuffer key = ByteBuffer.allocate(4); |
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString()); |
||||||
|
final RocksIterator it = db.newIterator()) { |
||||||
|
it.seekToFirst(); |
||||||
|
int lastKey = Integer.MIN_VALUE; |
||||||
|
int count = 0; |
||||||
|
for (it.seekToFirst(); it.isValid(); it.next()) { |
||||||
|
key.put(it.key()); |
||||||
|
key.flip(); |
||||||
|
final int thisKey = key.getInt(); |
||||||
|
key.clear(); |
||||||
|
assertThat(thisKey).isGreaterThan(lastKey); |
||||||
|
lastKey = thisKey; |
||||||
|
count++; |
||||||
|
} |
||||||
|
assertThat(count).isEqualTo(TOTAL_KEYS); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test which stores random keys into a column family |
||||||
|
* in the database |
||||||
|
* using an {@link IntComparator} |
||||||
|
* it then checks that these keys are read back in |
||||||
|
* ascending order |
||||||
|
* |
||||||
|
* @param db_path A path where we can store database |
||||||
|
* files temporarily |
||||||
|
* |
||||||
|
* @param comparator the comparator |
||||||
|
* |
||||||
|
* @throws RocksDBException if a database error happens. |
||||||
|
*/ |
||||||
|
private void testRoundtripCf(final Path db_path, |
||||||
|
final AbstractComparator comparator) throws RocksDBException { |
||||||
|
|
||||||
|
final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList( |
||||||
|
new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), |
||||||
|
new ColumnFamilyDescriptor("new_cf".getBytes(), |
||||||
|
new ColumnFamilyOptions() |
||||||
|
.setComparator(comparator)) |
||||||
|
); |
||||||
|
|
||||||
|
final List<ColumnFamilyHandle> cfHandles = new ArrayList<>(); |
||||||
|
|
||||||
|
try (final DBOptions opt = new DBOptions() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setCreateMissingColumnFamilies(true)) { |
||||||
|
|
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString(), |
||||||
|
cfDescriptors, cfHandles)) { |
||||||
|
try { |
||||||
|
assertThat(cfDescriptors.size()).isEqualTo(2); |
||||||
|
assertThat(cfHandles.size()).isEqualTo(2); |
||||||
|
|
||||||
|
for (int i = 0; i < TOTAL_KEYS; i++) { |
||||||
|
db.put(cfHandles.get(1), keys[i], "value".getBytes(UTF_8)); |
||||||
|
} |
||||||
|
} finally { |
||||||
|
for (final ColumnFamilyHandle cfHandle : cfHandles) { |
||||||
|
cfHandle.close(); |
||||||
|
} |
||||||
|
cfHandles.clear(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// re-open db and read from start to end
|
||||||
|
// integer keys should be in ascending
|
||||||
|
// order as defined by SimpleIntComparator
|
||||||
|
final ByteBuffer key = ByteBuffer.allocate(4); |
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString(), |
||||||
|
cfDescriptors, cfHandles); |
||||||
|
final RocksIterator it = db.newIterator(cfHandles.get(1))) { |
||||||
|
try { |
||||||
|
assertThat(cfDescriptors.size()).isEqualTo(2); |
||||||
|
assertThat(cfHandles.size()).isEqualTo(2); |
||||||
|
|
||||||
|
it.seekToFirst(); |
||||||
|
int lastKey = Integer.MIN_VALUE; |
||||||
|
int count = 0; |
||||||
|
for (it.seekToFirst(); it.isValid(); it.next()) { |
||||||
|
key.put(it.key()); |
||||||
|
key.flip(); |
||||||
|
final int thisKey = key.getInt(); |
||||||
|
key.clear(); |
||||||
|
assertThat(thisKey).isGreaterThan(lastKey); |
||||||
|
lastKey = thisKey; |
||||||
|
count++; |
||||||
|
} |
||||||
|
|
||||||
|
assertThat(count).isEqualTo(TOTAL_KEYS); |
||||||
|
|
||||||
|
} finally { |
||||||
|
for (final ColumnFamilyHandle cfHandle : cfHandles) { |
||||||
|
cfHandle.close(); |
||||||
|
} |
||||||
|
cfHandles.clear(); |
||||||
|
for (final ColumnFamilyDescriptor cfDescriptor : cfDescriptors) { |
||||||
|
cfDescriptor.getOptions().close(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,266 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb.util; |
||||||
|
|
||||||
|
import org.junit.BeforeClass; |
||||||
|
import org.junit.ClassRule; |
||||||
|
import org.junit.Rule; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.rules.TemporaryFolder; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.junit.runners.Parameterized; |
||||||
|
import org.junit.runners.Parameterized.Parameter; |
||||||
|
import org.junit.runners.Parameterized.Parameters; |
||||||
|
import org.rocksdb.*; |
||||||
|
|
||||||
|
import java.nio.ByteBuffer; |
||||||
|
import java.nio.file.*; |
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Arrays; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Random; |
||||||
|
|
||||||
|
import static java.nio.charset.StandardCharsets.UTF_8; |
||||||
|
import static org.assertj.core.api.Assertions.assertThat; |
||||||
|
|
||||||
|
/** |
||||||
|
* Tests for IntComparator, but more generally |
||||||
|
* also for rocksdb::ComparatorJniCallback implementation. |
||||||
|
*/ |
||||||
|
@RunWith(Parameterized.class) |
||||||
|
public class IntComparatorTest { |
||||||
|
|
||||||
|
// test with 500 random integer keys
|
||||||
|
private static final int TOTAL_KEYS = 500; |
||||||
|
private static final byte[][] keys = new byte[TOTAL_KEYS][4]; |
||||||
|
|
||||||
|
@BeforeClass |
||||||
|
public static void prepareKeys() { |
||||||
|
final ByteBuffer buf = ByteBuffer.allocate(4); |
||||||
|
final Random random = new Random(); |
||||||
|
for (int i = 0; i < TOTAL_KEYS; i++) { |
||||||
|
final int ri = random.nextInt(); |
||||||
|
buf.putInt(ri); |
||||||
|
buf.flip(); |
||||||
|
final byte[] key = buf.array(); |
||||||
|
|
||||||
|
// does key already exist (avoid duplicates)
|
||||||
|
if (keyExists(key, i)) { |
||||||
|
i--; // loop round and generate a different key
|
||||||
|
} else { |
||||||
|
System.arraycopy(key, 0, keys[i], 0, 4); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static boolean keyExists(final byte[] key, final int limit) { |
||||||
|
for (int j = 0; j < limit; j++) { |
||||||
|
if (Arrays.equals(key, keys[j])) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Parameters(name = "{0}") |
||||||
|
public static Iterable<Object[]> parameters() { |
||||||
|
return Arrays.asList(new Object[][] { |
||||||
|
{ "non-direct_reused64_mutex", false, 64, ReusedSynchronisationType.MUTEX }, |
||||||
|
{ "direct_reused64_mutex", true, 64, ReusedSynchronisationType.MUTEX }, |
||||||
|
{ "non-direct_reused64_adaptive-mutex", false, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, |
||||||
|
{ "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, |
||||||
|
{ "non-direct_reused64_thread-local", false, 64, ReusedSynchronisationType.THREAD_LOCAL }, |
||||||
|
{ "direct_reused64_thread-local", true, 64, ReusedSynchronisationType.THREAD_LOCAL }, |
||||||
|
{ "non-direct_noreuse", false, -1, null }, |
||||||
|
{ "direct_noreuse", true, -1, null } |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
@Parameter(0) |
||||||
|
public String name; |
||||||
|
|
||||||
|
@Parameter(1) |
||||||
|
public boolean useDirectBuffer; |
||||||
|
|
||||||
|
@Parameter(2) |
||||||
|
public int maxReusedBufferSize; |
||||||
|
|
||||||
|
@Parameter(3) |
||||||
|
public ReusedSynchronisationType reusedSynchronisationType; |
||||||
|
|
||||||
|
@ClassRule |
||||||
|
public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = |
||||||
|
new RocksNativeLibraryResource(); |
||||||
|
|
||||||
|
@Rule |
||||||
|
public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void javaComparatorDefaultCf() throws RocksDBException { |
||||||
|
try (final ComparatorOptions options = new ComparatorOptions() |
||||||
|
.setUseDirectBuffer(useDirectBuffer) |
||||||
|
.setMaxReusedBufferSize(maxReusedBufferSize) |
||||||
|
// if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used
|
||||||
|
.setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); |
||||||
|
final IntComparator comparator = new IntComparator(options)) { |
||||||
|
|
||||||
|
// test the round-tripability of keys written and read with the Comparator
|
||||||
|
testRoundtrip(FileSystems.getDefault().getPath( |
||||||
|
dbFolder.getRoot().getAbsolutePath()), comparator); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void javaComparatorNamedCf() throws RocksDBException { |
||||||
|
try (final ComparatorOptions options = new ComparatorOptions() |
||||||
|
.setUseDirectBuffer(useDirectBuffer) |
||||||
|
.setMaxReusedBufferSize(maxReusedBufferSize) |
||||||
|
// if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used
|
||||||
|
.setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); |
||||||
|
final IntComparator comparator = new IntComparator(options)) { |
||||||
|
|
||||||
|
// test the round-tripability of keys written and read with the Comparator
|
||||||
|
testRoundtripCf(FileSystems.getDefault().getPath( |
||||||
|
dbFolder.getRoot().getAbsolutePath()), comparator); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test which stores random keys into the database |
||||||
|
* using an {@link IntComparator} |
||||||
|
* it then checks that these keys are read back in |
||||||
|
* ascending order |
||||||
|
* |
||||||
|
* @param db_path A path where we can store database |
||||||
|
* files temporarily |
||||||
|
* |
||||||
|
* @param comparator the comparator |
||||||
|
* |
||||||
|
* @throws RocksDBException if a database error happens. |
||||||
|
*/ |
||||||
|
private void testRoundtrip(final Path db_path, |
||||||
|
final AbstractComparator comparator) throws RocksDBException { |
||||||
|
try (final Options opt = new Options() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setComparator(comparator)) { |
||||||
|
|
||||||
|
// store TOTAL_KEYS into the db
|
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString())) { |
||||||
|
for (int i = 0; i < TOTAL_KEYS; i++) { |
||||||
|
db.put(keys[i], "value".getBytes(UTF_8)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// re-open db and read from start to end
|
||||||
|
// integer keys should be in ascending
|
||||||
|
// order as defined by IntComparator
|
||||||
|
final ByteBuffer key = ByteBuffer.allocate(4); |
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString()); |
||||||
|
final RocksIterator it = db.newIterator()) { |
||||||
|
it.seekToFirst(); |
||||||
|
int lastKey = Integer.MIN_VALUE; |
||||||
|
int count = 0; |
||||||
|
for (it.seekToFirst(); it.isValid(); it.next()) { |
||||||
|
key.put(it.key()); |
||||||
|
key.flip(); |
||||||
|
final int thisKey = key.getInt(); |
||||||
|
key.clear(); |
||||||
|
assertThat(thisKey).isGreaterThan(lastKey); |
||||||
|
lastKey = thisKey; |
||||||
|
count++; |
||||||
|
} |
||||||
|
assertThat(count).isEqualTo(TOTAL_KEYS); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test which stores random keys into a column family |
||||||
|
* in the database |
||||||
|
* using an {@link IntComparator} |
||||||
|
* it then checks that these keys are read back in |
||||||
|
* ascending order |
||||||
|
* |
||||||
|
* @param db_path A path where we can store database |
||||||
|
* files temporarily |
||||||
|
* |
||||||
|
* @param comparator the comparator |
||||||
|
* |
||||||
|
* @throws RocksDBException if a database error happens. |
||||||
|
*/ |
||||||
|
private void testRoundtripCf(final Path db_path, |
||||||
|
final AbstractComparator comparator) throws RocksDBException { |
||||||
|
|
||||||
|
final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList( |
||||||
|
new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), |
||||||
|
new ColumnFamilyDescriptor("new_cf".getBytes(), |
||||||
|
new ColumnFamilyOptions() |
||||||
|
.setComparator(comparator)) |
||||||
|
); |
||||||
|
|
||||||
|
final List<ColumnFamilyHandle> cfHandles = new ArrayList<>(); |
||||||
|
|
||||||
|
try (final DBOptions opt = new DBOptions() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setCreateMissingColumnFamilies(true)) { |
||||||
|
|
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString(), |
||||||
|
cfDescriptors, cfHandles)) { |
||||||
|
try { |
||||||
|
assertThat(cfDescriptors.size()).isEqualTo(2); |
||||||
|
assertThat(cfHandles.size()).isEqualTo(2); |
||||||
|
|
||||||
|
for (int i = 0; i < TOTAL_KEYS; i++) { |
||||||
|
db.put(cfHandles.get(1), keys[i], "value".getBytes(UTF_8)); |
||||||
|
} |
||||||
|
} finally { |
||||||
|
for (final ColumnFamilyHandle cfHandle : cfHandles) { |
||||||
|
cfHandle.close(); |
||||||
|
} |
||||||
|
cfHandles.clear(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// re-open db and read from start to end
|
||||||
|
// integer keys should be in ascending
|
||||||
|
// order as defined by SimpleIntComparator
|
||||||
|
final ByteBuffer key = ByteBuffer.allocate(4); |
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString(), |
||||||
|
cfDescriptors, cfHandles); |
||||||
|
final RocksIterator it = db.newIterator(cfHandles.get(1))) { |
||||||
|
try { |
||||||
|
assertThat(cfDescriptors.size()).isEqualTo(2); |
||||||
|
assertThat(cfHandles.size()).isEqualTo(2); |
||||||
|
|
||||||
|
it.seekToFirst(); |
||||||
|
int lastKey = Integer.MIN_VALUE; |
||||||
|
int count = 0; |
||||||
|
for (it.seekToFirst(); it.isValid(); it.next()) { |
||||||
|
key.put(it.key()); |
||||||
|
key.flip(); |
||||||
|
final int thisKey = key.getInt(); |
||||||
|
key.clear(); |
||||||
|
assertThat(thisKey).isGreaterThan(lastKey); |
||||||
|
lastKey = thisKey; |
||||||
|
count++; |
||||||
|
} |
||||||
|
|
||||||
|
assertThat(count).isEqualTo(TOTAL_KEYS); |
||||||
|
|
||||||
|
} finally { |
||||||
|
for (final ColumnFamilyHandle cfHandle : cfHandles) { |
||||||
|
cfHandle.close(); |
||||||
|
} |
||||||
|
cfHandles.clear(); |
||||||
|
for (final ColumnFamilyDescriptor cfDescriptor : cfDescriptors) { |
||||||
|
cfDescriptor.getOptions().close(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,174 @@ |
|||||||
|
package org.rocksdb.util; |
||||||
|
|
||||||
|
import org.junit.ClassRule; |
||||||
|
import org.junit.Rule; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.rules.TemporaryFolder; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.junit.runners.Parameterized; |
||||||
|
import org.junit.runners.Parameterized.Parameter; |
||||||
|
import org.junit.runners.Parameterized.Parameters; |
||||||
|
import org.rocksdb.*; |
||||||
|
|
||||||
|
import java.io.IOException; |
||||||
|
import java.nio.ByteBuffer; |
||||||
|
import java.nio.file.*; |
||||||
|
import java.util.Arrays; |
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat; |
||||||
|
|
||||||
|
@RunWith(Parameterized.class) |
||||||
|
public class JNIComparatorTest { |
||||||
|
|
||||||
|
@Parameters(name = "{0}") |
||||||
|
public static Iterable<Object[]> parameters() { |
||||||
|
return Arrays.asList(new Object[][] { |
||||||
|
{ "bytewise_non-direct", BuiltinComparator.BYTEWISE_COMPARATOR, false }, |
||||||
|
{ "bytewise_direct", BuiltinComparator.BYTEWISE_COMPARATOR, true }, |
||||||
|
{ "reverse-bytewise_non-direct", BuiltinComparator.REVERSE_BYTEWISE_COMPARATOR, false }, |
||||||
|
{ "reverse-bytewise_direct", BuiltinComparator.REVERSE_BYTEWISE_COMPARATOR, true }, |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
@Parameter(0) |
||||||
|
public String name; |
||||||
|
|
||||||
|
@Parameter(1) |
||||||
|
public BuiltinComparator builtinComparator; |
||||||
|
|
||||||
|
@Parameter(2) |
||||||
|
public boolean useDirectBuffer; |
||||||
|
|
||||||
|
@ClassRule |
||||||
|
public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = |
||||||
|
new RocksNativeLibraryResource(); |
||||||
|
|
||||||
|
@Rule |
||||||
|
public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||||
|
|
||||||
|
private static final int MIN = Short.MIN_VALUE - 1; |
||||||
|
private static final int MAX = Short.MAX_VALUE + 1; |
||||||
|
|
||||||
|
@Test |
||||||
|
public void java_comparator_equals_cpp_comparator() throws RocksDBException, IOException { |
||||||
|
final int[] javaKeys; |
||||||
|
try (final ComparatorOptions comparatorOptions = new ComparatorOptions(); |
||||||
|
final AbstractComparator comparator = builtinComparator == BuiltinComparator.BYTEWISE_COMPARATOR |
||||||
|
? new BytewiseComparator(comparatorOptions) |
||||||
|
: new ReverseBytewiseComparator(comparatorOptions)) { |
||||||
|
final Path javaDbDir = |
||||||
|
FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); |
||||||
|
storeWithJavaComparator(javaDbDir, comparator); |
||||||
|
javaKeys = readAllWithJavaComparator(javaDbDir, comparator); |
||||||
|
} |
||||||
|
|
||||||
|
final Path cppDbDir = |
||||||
|
FileSystems.getDefault().getPath(dbFolder.newFolder().getAbsolutePath()); |
||||||
|
storeWithCppComparator(cppDbDir, builtinComparator); |
||||||
|
final int[] cppKeys = |
||||||
|
readAllWithCppComparator(cppDbDir, builtinComparator); |
||||||
|
|
||||||
|
assertThat(javaKeys).isEqualTo(cppKeys); |
||||||
|
} |
||||||
|
|
||||||
|
private void storeWithJavaComparator(final Path dir, |
||||||
|
final AbstractComparator comparator) throws RocksDBException { |
||||||
|
final ByteBuffer buf = ByteBuffer.allocate(4); |
||||||
|
try (final Options options = new Options() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setComparator(comparator); |
||||||
|
final RocksDB db = |
||||||
|
RocksDB.open(options, dir.toAbsolutePath().toString())) { |
||||||
|
for (int i = MIN; i < MAX; i++) { |
||||||
|
buf.putInt(i); |
||||||
|
buf.flip(); |
||||||
|
|
||||||
|
db.put(buf.array(), buf.array()); |
||||||
|
|
||||||
|
buf.clear(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private void storeWithCppComparator(final Path dir, |
||||||
|
final BuiltinComparator builtinComparator) throws RocksDBException { |
||||||
|
try (final Options options = new Options() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setComparator(builtinComparator); |
||||||
|
final RocksDB db = |
||||||
|
RocksDB.open(options, dir.toAbsolutePath().toString())) { |
||||||
|
|
||||||
|
final ByteBuffer buf = ByteBuffer.allocate(4); |
||||||
|
for (int i = MIN; i < MAX; i++) { |
||||||
|
buf.putInt(i); |
||||||
|
buf.flip(); |
||||||
|
|
||||||
|
db.put(buf.array(), buf.array()); |
||||||
|
|
||||||
|
buf.clear(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private int[] readAllWithJavaComparator(final Path dir, |
||||||
|
final AbstractComparator comparator) throws RocksDBException { |
||||||
|
try (final Options options = new Options() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setComparator(comparator); |
||||||
|
final RocksDB db = |
||||||
|
RocksDB.open(options, dir.toAbsolutePath().toString())) { |
||||||
|
|
||||||
|
try (final RocksIterator it = db.newIterator()) { |
||||||
|
it.seekToFirst(); |
||||||
|
|
||||||
|
final ByteBuffer buf = ByteBuffer.allocate(4); |
||||||
|
final int[] keys = new int[MAX - MIN]; |
||||||
|
int idx = 0; |
||||||
|
while (it.isValid()) { |
||||||
|
buf.put(it.key()); |
||||||
|
buf.flip(); |
||||||
|
|
||||||
|
final int thisKey = buf.getInt(); |
||||||
|
keys[idx++] = thisKey; |
||||||
|
|
||||||
|
buf.clear(); |
||||||
|
|
||||||
|
it.next(); |
||||||
|
} |
||||||
|
|
||||||
|
return keys; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private int[] readAllWithCppComparator(final Path dir, |
||||||
|
final BuiltinComparator comparator) throws RocksDBException { |
||||||
|
try (final Options options = new Options() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setComparator(comparator); |
||||||
|
final RocksDB db = |
||||||
|
RocksDB.open(options, dir.toAbsolutePath().toString())) { |
||||||
|
|
||||||
|
try (final RocksIterator it = db.newIterator()) { |
||||||
|
it.seekToFirst(); |
||||||
|
|
||||||
|
final ByteBuffer buf = ByteBuffer.allocate(4); |
||||||
|
final int[] keys = new int[MAX - MIN]; |
||||||
|
int idx = 0; |
||||||
|
while (it.isValid()) { |
||||||
|
buf.put(it.key()); |
||||||
|
buf.flip(); |
||||||
|
|
||||||
|
final int thisKey = buf.getInt(); |
||||||
|
keys[idx++] = thisKey; |
||||||
|
|
||||||
|
buf.clear(); |
||||||
|
|
||||||
|
it.next(); |
||||||
|
} |
||||||
|
|
||||||
|
return keys; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,270 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
package org.rocksdb.util; |
||||||
|
|
||||||
|
import org.junit.BeforeClass; |
||||||
|
import org.junit.ClassRule; |
||||||
|
import org.junit.Rule; |
||||||
|
import org.junit.Test; |
||||||
|
import org.junit.rules.TemporaryFolder; |
||||||
|
import org.junit.runner.RunWith; |
||||||
|
import org.junit.runners.Parameterized; |
||||||
|
import org.junit.runners.Parameterized.Parameter; |
||||||
|
import org.junit.runners.Parameterized.Parameters; |
||||||
|
import org.rocksdb.*; |
||||||
|
|
||||||
|
import java.nio.ByteBuffer; |
||||||
|
import java.nio.file.FileSystems; |
||||||
|
import java.nio.file.Path; |
||||||
|
import java.util.ArrayList; |
||||||
|
import java.util.Arrays; |
||||||
|
import java.util.List; |
||||||
|
import java.util.Random; |
||||||
|
|
||||||
|
import static java.nio.charset.StandardCharsets.UTF_8; |
||||||
|
import static org.assertj.core.api.Assertions.assertThat; |
||||||
|
|
||||||
|
/** |
||||||
|
* Similar to {@link IntComparatorTest}, but uses |
||||||
|
* {@link ReverseBytewiseComparator} which ensures the correct reverse |
||||||
|
* ordering of positive integers. |
||||||
|
*/ |
||||||
|
@RunWith(Parameterized.class) |
||||||
|
public class ReverseBytewiseComparatorIntTest { |
||||||
|
|
||||||
|
// test with 500 random positive integer keys
|
||||||
|
private static final int TOTAL_KEYS = 500; |
||||||
|
private static final byte[][] keys = new byte[TOTAL_KEYS][4]; |
||||||
|
|
||||||
|
@BeforeClass |
||||||
|
public static void prepareKeys() { |
||||||
|
final ByteBuffer buf = ByteBuffer.allocate(4); |
||||||
|
final Random random = new Random(); |
||||||
|
for (int i = 0; i < TOTAL_KEYS; i++) { |
||||||
|
final int ri = random.nextInt() & Integer.MAX_VALUE; // the & ensures positive integer
|
||||||
|
buf.putInt(ri); |
||||||
|
buf.flip(); |
||||||
|
final byte[] key = buf.array(); |
||||||
|
|
||||||
|
// does key already exist (avoid duplicates)
|
||||||
|
if (keyExists(key, i)) { |
||||||
|
i--; // loop round and generate a different key
|
||||||
|
} else { |
||||||
|
System.arraycopy(key, 0, keys[i], 0, 4); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private static boolean keyExists(final byte[] key, final int limit) { |
||||||
|
for (int j = 0; j < limit; j++) { |
||||||
|
if (Arrays.equals(key, keys[j])) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Parameters(name = "{0}") |
||||||
|
public static Iterable<Object[]> parameters() { |
||||||
|
return Arrays.asList(new Object[][] { |
||||||
|
{ "non-direct_reused64_mutex", false, 64, ReusedSynchronisationType.MUTEX }, |
||||||
|
{ "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.MUTEX }, |
||||||
|
{ "non-direct_reused64_adaptive-mutex", false, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, |
||||||
|
{ "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.ADAPTIVE_MUTEX }, |
||||||
|
{ "non-direct_reused64_adaptive-mutex", false, 64, ReusedSynchronisationType.THREAD_LOCAL }, |
||||||
|
{ "direct_reused64_adaptive-mutex", true, 64, ReusedSynchronisationType.THREAD_LOCAL }, |
||||||
|
{ "non-direct_noreuse", false, -1, null }, |
||||||
|
{ "direct_noreuse", true, -1, null } |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
@Parameter(0) |
||||||
|
public String name; |
||||||
|
|
||||||
|
@Parameter(1) |
||||||
|
public boolean useDirectBuffer; |
||||||
|
|
||||||
|
@Parameter(2) |
||||||
|
public int maxReusedBufferSize; |
||||||
|
|
||||||
|
@Parameter(3) |
||||||
|
public ReusedSynchronisationType reusedSynchronisationType; |
||||||
|
|
||||||
|
@ClassRule |
||||||
|
public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE = |
||||||
|
new RocksNativeLibraryResource(); |
||||||
|
|
||||||
|
@Rule |
||||||
|
public TemporaryFolder dbFolder = new TemporaryFolder(); |
||||||
|
|
||||||
|
|
||||||
|
@Test |
||||||
|
public void javaComparatorDefaultCf() throws RocksDBException { |
||||||
|
try (final ComparatorOptions options = new ComparatorOptions() |
||||||
|
.setUseDirectBuffer(useDirectBuffer) |
||||||
|
.setMaxReusedBufferSize(maxReusedBufferSize) |
||||||
|
// if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used
|
||||||
|
.setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); |
||||||
|
final ReverseBytewiseComparator comparator = |
||||||
|
new ReverseBytewiseComparator(options)) { |
||||||
|
|
||||||
|
// test the round-tripability of keys written and read with the Comparator
|
||||||
|
testRoundtrip(FileSystems.getDefault().getPath( |
||||||
|
dbFolder.getRoot().getAbsolutePath()), comparator); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
public void javaComparatorNamedCf() throws RocksDBException { |
||||||
|
try (final ComparatorOptions options = new ComparatorOptions() |
||||||
|
.setUseDirectBuffer(useDirectBuffer) |
||||||
|
.setMaxReusedBufferSize(maxReusedBufferSize) |
||||||
|
// if reusedSynchronisationType == null we assume that maxReusedBufferSize <= 0 and so we just set ADAPTIVE_MUTEX, even though it won't be used
|
||||||
|
.setReusedSynchronisationType(reusedSynchronisationType == null ? ReusedSynchronisationType.ADAPTIVE_MUTEX : reusedSynchronisationType); |
||||||
|
final ReverseBytewiseComparator comparator |
||||||
|
= new ReverseBytewiseComparator(options)) { |
||||||
|
|
||||||
|
// test the round-tripability of keys written and read with the Comparator
|
||||||
|
testRoundtripCf(FileSystems.getDefault().getPath( |
||||||
|
dbFolder.getRoot().getAbsolutePath()), comparator); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test which stores random keys into the database |
||||||
|
* using an {@link IntComparator} |
||||||
|
* it then checks that these keys are read back in |
||||||
|
* ascending order |
||||||
|
* |
||||||
|
* @param db_path A path where we can store database |
||||||
|
* files temporarily |
||||||
|
* |
||||||
|
* @param comparator the comparator |
||||||
|
* |
||||||
|
* @throws RocksDBException if a database error happens. |
||||||
|
*/ |
||||||
|
private void testRoundtrip(final Path db_path, |
||||||
|
final AbstractComparator comparator) throws RocksDBException { |
||||||
|
try (final Options opt = new Options() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setComparator(comparator)) { |
||||||
|
|
||||||
|
// store TOTAL_KEYS into the db
|
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString())) { |
||||||
|
for (int i = 0; i < TOTAL_KEYS; i++) { |
||||||
|
db.put(keys[i], "value".getBytes(UTF_8)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// re-open db and read from start to end
|
||||||
|
// integer keys should be in descending
|
||||||
|
// order
|
||||||
|
final ByteBuffer key = ByteBuffer.allocate(4); |
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString()); |
||||||
|
final RocksIterator it = db.newIterator()) { |
||||||
|
it.seekToFirst(); |
||||||
|
int lastKey = Integer.MAX_VALUE; |
||||||
|
int count = 0; |
||||||
|
for (it.seekToFirst(); it.isValid(); it.next()) { |
||||||
|
key.put(it.key()); |
||||||
|
key.flip(); |
||||||
|
final int thisKey = key.getInt(); |
||||||
|
key.clear(); |
||||||
|
assertThat(thisKey).isLessThan(lastKey); |
||||||
|
lastKey = thisKey; |
||||||
|
count++; |
||||||
|
} |
||||||
|
assertThat(count).isEqualTo(TOTAL_KEYS); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Test which stores random keys into a column family |
||||||
|
* in the database |
||||||
|
* using an {@link IntComparator} |
||||||
|
* it then checks that these keys are read back in |
||||||
|
* ascending order |
||||||
|
* |
||||||
|
* @param db_path A path where we can store database |
||||||
|
* files temporarily |
||||||
|
* |
||||||
|
* @param comparator the comparator |
||||||
|
* |
||||||
|
* @throws RocksDBException if a database error happens. |
||||||
|
*/ |
||||||
|
private void testRoundtripCf(final Path db_path, |
||||||
|
final AbstractComparator comparator) throws RocksDBException { |
||||||
|
|
||||||
|
final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList( |
||||||
|
new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY), |
||||||
|
new ColumnFamilyDescriptor("new_cf".getBytes(), |
||||||
|
new ColumnFamilyOptions() |
||||||
|
.setComparator(comparator)) |
||||||
|
); |
||||||
|
|
||||||
|
final List<ColumnFamilyHandle> cfHandles = new ArrayList<>(); |
||||||
|
|
||||||
|
try (final DBOptions opt = new DBOptions() |
||||||
|
.setCreateIfMissing(true) |
||||||
|
.setCreateMissingColumnFamilies(true)) { |
||||||
|
|
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString(), |
||||||
|
cfDescriptors, cfHandles)) { |
||||||
|
try { |
||||||
|
assertThat(cfDescriptors.size()).isEqualTo(2); |
||||||
|
assertThat(cfHandles.size()).isEqualTo(2); |
||||||
|
|
||||||
|
for (int i = 0; i < TOTAL_KEYS; i++) { |
||||||
|
db.put(cfHandles.get(1), keys[i], "value".getBytes(UTF_8)); |
||||||
|
} |
||||||
|
} finally { |
||||||
|
for (final ColumnFamilyHandle cfHandle : cfHandles) { |
||||||
|
cfHandle.close(); |
||||||
|
} |
||||||
|
cfHandles.clear(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// re-open db and read from start to end
|
||||||
|
// integer keys should be in descending
|
||||||
|
// order
|
||||||
|
final ByteBuffer key = ByteBuffer.allocate(4); |
||||||
|
try (final RocksDB db = RocksDB.open(opt, db_path.toString(), |
||||||
|
cfDescriptors, cfHandles); |
||||||
|
final RocksIterator it = db.newIterator(cfHandles.get(1))) { |
||||||
|
try { |
||||||
|
assertThat(cfDescriptors.size()).isEqualTo(2); |
||||||
|
assertThat(cfHandles.size()).isEqualTo(2); |
||||||
|
|
||||||
|
it.seekToFirst(); |
||||||
|
int lastKey = Integer.MAX_VALUE; |
||||||
|
int count = 0; |
||||||
|
for (it.seekToFirst(); it.isValid(); it.next()) { |
||||||
|
key.put(it.key()); |
||||||
|
key.flip(); |
||||||
|
final int thisKey = key.getInt(); |
||||||
|
key.clear(); |
||||||
|
assertThat(thisKey).isLessThan(lastKey); |
||||||
|
lastKey = thisKey; |
||||||
|
count++; |
||||||
|
} |
||||||
|
|
||||||
|
assertThat(count).isEqualTo(TOTAL_KEYS); |
||||||
|
|
||||||
|
} finally { |
||||||
|
for (final ColumnFamilyHandle cfHandle : cfHandles) { |
||||||
|
cfHandle.close(); |
||||||
|
} |
||||||
|
cfHandles.clear(); |
||||||
|
for (final ColumnFamilyDescriptor cfDescriptor : cfDescriptors) { |
||||||
|
cfDescriptor.getOptions().close(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
Loading…
Reference in new issue