@ -601,48 +601,73 @@ public class RocksDBTest {
RocksDB db = null ;
Options opt = null ;
try {
final int NUM_KEYS_PER_L0_FILE = 100 ;
final int KEY_SIZE = 20 ;
final int VALUE_SIZE = 300 ;
final int L0_FILE_SIZE =
NUM_KEYS_PER_L0_FILE * ( KEY_SIZE + VALUE_SIZE ) ;
final int NUM_L0_FILES = 10 ;
final int TEST_SCALE = 5 ;
final int KEY_INTERVAL = 100 ;
opt = new Options ( ) .
setCreateIfMissing ( true ) .
setCompactionStyle ( CompactionStyle . LEVEL ) .
setNumLevels ( 4 ) .
setWriteBufferSize ( 100 < < 10 ) .
setLevelZeroFileNumCompactionTrigger ( 3 ) .
setTargetFileSizeBase ( 200 < < 10 ) .
setTargetFileSizeMultiplier ( 1 ) .
setMaxBytesForLevelBase ( 500 < < 10 ) .
setMaxBytesForLevelMultiplier ( 1 ) .
setDisableAutoCompactions ( false ) ;
// open database
db = RocksDB . open ( opt ,
dbFolder . getRoot ( ) . getAbsolutePath ( ) ) ;
// fill database with key/value pairs
byte [ ] b = new byte [ 10000 ] ;
for ( int i = 0 ; i < 200 ; i + + ) {
rand . nextBytes ( b ) ;
db . put ( ( String . valueOf ( i ) ) . getBytes ( ) , b ) ;
}
db . flush ( new FlushOptions ( ) . setWaitForFlush ( true ) ) ;
db . close ( ) ;
opt . setTargetFileSizeBase ( Long . MAX_VALUE ) .
setNumLevels ( 5 ) .
// a slightly bigger write buffer than L0 file
// so that we can ensure manual flush always
// go before background flush happens.
setWriteBufferSize ( L0_FILE_SIZE * 2 ) .
// Disable auto L0 -> L1 compaction
setLevelZeroFileNumCompactionTrigger ( 20 ) .
setTargetFileSizeBase ( L0_FILE_SIZE * 100 ) .
setTargetFileSizeMultiplier ( 1 ) .
setMaxBytesForLevelBase ( Long . MAX_VALUE ) .
setMaxBytesForLevelMultiplier ( 1 ) .
// To disable auto compaction
setMaxBytesForLevelBase ( NUM_L0_FILES * L0_FILE_SIZE * 100 ) .
setMaxBytesForLevelMultiplier ( 2 ) .
setDisableAutoCompactions ( true ) ;
db = RocksDB . open ( opt ,
dbFolder . getRoot ( ) . getAbsolutePath ( ) ) ;
// fill database with key/value pairs
byte [ ] value = new byte [ VALUE_SIZE ] ;
int int_key = 0 ;
for ( int round = 0 ; round < 5 ; + + round ) {
int initial_key = int_key ;
for ( int f = 1 ; f < = NUM_L0_FILES ; + + f ) {
for ( int i = 0 ; i < NUM_KEYS_PER_L0_FILE ; + + i ) {
int_key + = KEY_INTERVAL ;
rand . nextBytes ( value ) ;
db . compactRange ( true , 0 , 0 ) ;
for ( int i = 0 ; i < 4 ; i + + ) {
if ( i = = 0 ) {
assertThat (
db . getProperty ( "rocksdb.num-files-at-level" + i ) ) .
isEqualTo ( "1" ) ;
} else {
db . put ( String . format ( "%020d" , int_key ) . getBytes ( ) ,
value ) ;
}
db . flush ( new FlushOptions ( ) . setWaitForFlush ( true ) ) ;
// Make sure we do create one more L0 files.
assertThat (
db . getProperty ( "rocksdb.num-files-at-level" + i ) ) .
isEqualTo ( "0 " ) ;
db . getProperty ( "rocksdb.num-files-at-level0" ) ) .
isEqualTo ( "" + f ) ;
}
// Compact all L0 files we just created
db . compactRange (
String . format ( "%020d" , initial_key ) . getBytes ( ) ,
String . format ( "%020d" , int_key - 1 ) . getBytes ( ) ) ;
// Making sure there isn't any L0 files.
assertThat (
db . getProperty ( "rocksdb.num-files-at-level0" ) ) .
isEqualTo ( "0" ) ;
// Making sure there are some L1 files.
// Here we only use != 0 instead of a specific number
// as we don't want the test make any assumption on
// how compaction works.
assertThat (
db . getProperty ( "rocksdb.num-files-at-level1" ) ) .
isNotEqualTo ( "0" ) ;
// Because we only compacted those keys we issued
// in this round, there shouldn't be any L1 -> L2
// compaction. So we expect zero L2 files here.
assertThat (
db . getProperty ( "rocksdb.num-files-at-level2" ) ) .
isEqualTo ( "0" ) ;
}
} finally {
if ( db ! = null ) {
@ -662,6 +687,14 @@ public class RocksDBTest {
List < ColumnFamilyHandle > columnFamilyHandles =
new ArrayList < > ( ) ;
try {
final int NUM_KEYS_PER_L0_FILE = 100 ;
final int KEY_SIZE = 20 ;
final int VALUE_SIZE = 300 ;
final int L0_FILE_SIZE =
NUM_KEYS_PER_L0_FILE * ( KEY_SIZE + VALUE_SIZE ) ;
final int NUM_L0_FILES = 10 ;
final int TEST_SCALE = 5 ;
final int KEY_INTERVAL = 100 ;
opt = new DBOptions ( ) .
setCreateIfMissing ( true ) .
setCreateMissingColumnFamilies ( true ) ;
@ -672,62 +705,73 @@ public class RocksDBTest {
columnFamilyDescriptors . add ( new ColumnFamilyDescriptor (
"new_cf" . getBytes ( ) ,
new ColumnFamilyOptions ( ) .
setDisableAutoCompactions ( true ) .
setCompactionStyle ( CompactionStyle . LEVEL ) .
setNumLevels ( 4 ) .
setWriteBufferSize ( 100 < < 10 ) .
setLevelZeroFileNumCompactionTrigger ( 3 ) .
setTargetFileSizeBase ( 200 < < 10 ) .
setNumLevels ( 5 ) .
// a slightly bigger write buffer than L0 file
// so that we can ensure manual flush always
// go before background flush happens.
setWriteBufferSize ( L0_FILE_SIZE * 2 ) .
// Disable auto L0 -> L1 compaction
setLevelZeroFileNumCompactionTrigger ( 20 ) .
setTargetFileSizeBase ( L0_FILE_SIZE * 100 ) .
setTargetFileSizeMultiplier ( 1 ) .
setMaxBytesForLevelBase ( 500 < < 10 ) .
setMaxBytesForLevelMultiplier ( 1 ) .
setDisableAutoCompactions ( false ) ) ) ;
// To disable auto compaction
setMaxBytesForLevelBase ( NUM_L0_FILES * L0_FILE_SIZE * 100 ) .
setMaxBytesForLevelMultiplier ( 2 ) .
setDisableAutoCompactions ( true ) ) ) ;
// open database
db = RocksDB . open ( opt ,
dbFolder . getRoot ( ) . getAbsolutePath ( ) ,
columnFamilyDescriptors ,
columnFamilyHandles ) ;
// fill database with key/value pairs
byte [ ] b = new byte [ 10000 ] ;
for ( int i = 0 ; i < 200 ; i + + ) {
rand . nextBytes ( b ) ;
db . put ( columnFamilyHandles . get ( 1 ) ,
String . valueOf ( i ) . getBytes ( ) , b ) ;
}
db . flush ( new FlushOptions ( ) . setWaitForFlush ( true ) ,
columnFamilyHandles . get ( 1 ) ) ;
// free column families
for ( ColumnFamilyHandle handle : columnFamilyHandles ) {
handle . dispose ( ) ;
}
// clear column family handles for reopen
columnFamilyHandles . clear ( ) ;
db . close ( ) ;
columnFamilyDescriptors . get ( 1 ) .
columnFamilyOptions ( ) .
setTargetFileSizeBase ( Long . MAX_VALUE ) .
setTargetFileSizeMultiplier ( 1 ) .
setMaxBytesForLevelBase ( Long . MAX_VALUE ) .
setMaxBytesForLevelMultiplier ( 1 ) .
setDisableAutoCompactions ( true ) ;
// reopen database
db = RocksDB . open ( opt ,
dbFolder . getRoot ( ) . getAbsolutePath ( ) ,
columnFamilyDescriptors ,
columnFamilyHandles ) ;
// compact new column family
db . compactRange ( columnFamilyHandles . get ( 1 ) , true , 0 , 0 ) ;
// check if new column family is compacted to level zero
for ( int i = 0 ; i < 4 ; i + + ) {
if ( i = = 0 ) {
assertThat ( db . getProperty ( columnFamilyHandles . get ( 1 ) ,
"rocksdb.num-files-at-level" + i ) ) .
isEqualTo ( "1" ) ;
} else {
assertThat ( db . getProperty ( columnFamilyHandles . get ( 1 ) ,
"rocksdb.num-files-at-level" + i ) ) .
isEqualTo ( "0" ) ;
byte [ ] value = new byte [ VALUE_SIZE ] ;
int int_key = 0 ;
for ( int round = 0 ; round < 5 ; + + round ) {
int initial_key = int_key ;
for ( int f = 1 ; f < = NUM_L0_FILES ; + + f ) {
for ( int i = 0 ; i < NUM_KEYS_PER_L0_FILE ; + + i ) {
int_key + = KEY_INTERVAL ;
rand . nextBytes ( value ) ;
db . put ( columnFamilyHandles . get ( 1 ) ,
String . format ( "%020d" , int_key ) . getBytes ( ) ,
value ) ;
}
db . flush ( new FlushOptions ( ) . setWaitForFlush ( true ) ,
columnFamilyHandles . get ( 1 ) ) ;
// Make sure we do create one more L0 files.
assertThat (
db . getProperty ( columnFamilyHandles . get ( 1 ) ,
"rocksdb.num-files-at-level0" ) ) .
isEqualTo ( "" + f ) ;
}
// Compact all L0 files we just created
db . compactRange (
columnFamilyHandles . get ( 1 ) ,
String . format ( "%020d" , initial_key ) . getBytes ( ) ,
String . format ( "%020d" , int_key - 1 ) . getBytes ( ) ) ;
// Making sure there isn't any L0 files.
assertThat (
db . getProperty ( columnFamilyHandles . get ( 1 ) ,
"rocksdb.num-files-at-level0" ) ) .
isEqualTo ( "0" ) ;
// Making sure there are some L1 files.
// Here we only use != 0 instead of a specific number
// as we don't want the test make any assumption on
// how compaction works.
assertThat (
db . getProperty ( columnFamilyHandles . get ( 1 ) ,
"rocksdb.num-files-at-level1" ) ) .
isNotEqualTo ( "0" ) ;
// Because we only compacted those keys we issued
// in this round, there shouldn't be any L1 -> L2
// compaction. So we expect zero L2 files here.
assertThat (
db . getProperty ( columnFamilyHandles . get ( 1 ) ,
"rocksdb.num-files-at-level2" ) ) .
isEqualTo ( "0" ) ;
}
} finally {
for ( ColumnFamilyHandle handle : columnFamilyHandles ) {