From c00948d5e14298712dc154881e79a95cd3f57b47 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Wed, 1 Jul 2015 16:03:56 -0700 Subject: [PATCH] [RocksJava] Fix test failure of compactRangeToLevel Summary: Rewrite Java tests compactRangeToLevel and compactRangeToLevelColumnFamily to make them more deterministic and robust. Test Plan: make rocksdbjava make jtest Reviewers: anthony, fyrz, adamretter, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D40941 --- .../test/java/org/rocksdb/RocksDBTest.java | 202 +++++++++++------- 1 file changed, 123 insertions(+), 79 deletions(-) diff --git a/java/src/test/java/org/rocksdb/RocksDBTest.java b/java/src/test/java/org/rocksdb/RocksDBTest.java index ae83683d6..31d2c5238 100644 --- a/java/src/test/java/org/rocksdb/RocksDBTest.java +++ b/java/src/test/java/org/rocksdb/RocksDBTest.java @@ -601,48 +601,73 @@ public class RocksDBTest { RocksDB db = null; Options opt = null; try { + final int NUM_KEYS_PER_L0_FILE = 100; + final int KEY_SIZE = 20; + final int VALUE_SIZE = 300; + final int L0_FILE_SIZE = + NUM_KEYS_PER_L0_FILE * (KEY_SIZE + VALUE_SIZE); + final int NUM_L0_FILES = 10; + final int TEST_SCALE = 5; + final int KEY_INTERVAL = 100; opt = new Options(). setCreateIfMissing(true). setCompactionStyle(CompactionStyle.LEVEL). - setNumLevels(4). - setWriteBufferSize(100<<10). - setLevelZeroFileNumCompactionTrigger(3). - setTargetFileSizeBase(200 << 10). - setTargetFileSizeMultiplier(1). - setMaxBytesForLevelBase(500 << 10). - setMaxBytesForLevelMultiplier(1). - setDisableAutoCompactions(false); - // open database - db = RocksDB.open(opt, - dbFolder.getRoot().getAbsolutePath()); - // fill database with key/value pairs - byte[] b = new byte[10000]; - for (int i = 0; i < 200; i++) { - rand.nextBytes(b); - db.put((String.valueOf(i)).getBytes(), b); - } - db.flush(new FlushOptions().setWaitForFlush(true)); - db.close(); - opt.setTargetFileSizeBase(Long.MAX_VALUE). + setNumLevels(5). + // a slightly bigger write buffer than L0 file + // so that we can ensure manual flush always + // go before background flush happens. + setWriteBufferSize(L0_FILE_SIZE * 2). + // Disable auto L0 -> L1 compaction + setLevelZeroFileNumCompactionTrigger(20). + setTargetFileSizeBase(L0_FILE_SIZE * 100). setTargetFileSizeMultiplier(1). - setMaxBytesForLevelBase(Long.MAX_VALUE). - setMaxBytesForLevelMultiplier(1). + // To disable auto compaction + setMaxBytesForLevelBase(NUM_L0_FILES * L0_FILE_SIZE * 100). + setMaxBytesForLevelMultiplier(2). setDisableAutoCompactions(true); - db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath()); + // fill database with key/value pairs + byte[] value = new byte[VALUE_SIZE]; + int int_key = 0; + for (int round = 0; round < 5; ++round) { + int initial_key = int_key; + for (int f = 1; f <= NUM_L0_FILES; ++f) { + for (int i = 0; i < NUM_KEYS_PER_L0_FILE; ++i) { + int_key += KEY_INTERVAL; + rand.nextBytes(value); - db.compactRange(true, 0, 0); - for (int i = 0; i < 4; i++) { - if (i == 0) { - assertThat( - db.getProperty("rocksdb.num-files-at-level" + i)). - isEqualTo("1"); - } else { + db.put(String.format("%020d", int_key).getBytes(), + value); + } + db.flush(new FlushOptions().setWaitForFlush(true)); + // Make sure we do create one more L0 files. assertThat( - db.getProperty("rocksdb.num-files-at-level" + i)). - isEqualTo("0"); + db.getProperty("rocksdb.num-files-at-level0")). + isEqualTo("" + f); } + + // Compact all L0 files we just created + db.compactRange( + String.format("%020d", initial_key).getBytes(), + String.format("%020d", int_key - 1).getBytes()); + // Making sure there isn't any L0 files. + assertThat( + db.getProperty("rocksdb.num-files-at-level0")). + isEqualTo("0"); + // Making sure there are some L1 files. + // Here we only use != 0 instead of a specific number + // as we don't want the test make any assumption on + // how compaction works. + assertThat( + db.getProperty("rocksdb.num-files-at-level1")). + isNotEqualTo("0"); + // Because we only compacted those keys we issued + // in this round, there shouldn't be any L1 -> L2 + // compaction. So we expect zero L2 files here. + assertThat( + db.getProperty("rocksdb.num-files-at-level2")). + isEqualTo("0"); } } finally { if (db != null) { @@ -662,6 +687,14 @@ public class RocksDBTest { List columnFamilyHandles = new ArrayList<>(); try { + final int NUM_KEYS_PER_L0_FILE = 100; + final int KEY_SIZE = 20; + final int VALUE_SIZE = 300; + final int L0_FILE_SIZE = + NUM_KEYS_PER_L0_FILE * (KEY_SIZE + VALUE_SIZE); + final int NUM_L0_FILES = 10; + final int TEST_SCALE = 5; + final int KEY_INTERVAL = 100; opt = new DBOptions(). setCreateIfMissing(true). setCreateMissingColumnFamilies(true); @@ -672,62 +705,73 @@ public class RocksDBTest { columnFamilyDescriptors.add(new ColumnFamilyDescriptor( "new_cf".getBytes(), new ColumnFamilyOptions(). - setDisableAutoCompactions(true). setCompactionStyle(CompactionStyle.LEVEL). - setNumLevels(4). - setWriteBufferSize(100 << 10). - setLevelZeroFileNumCompactionTrigger(3). - setTargetFileSizeBase(200 << 10). + setNumLevels(5). + // a slightly bigger write buffer than L0 file + // so that we can ensure manual flush always + // go before background flush happens. + setWriteBufferSize(L0_FILE_SIZE * 2). + // Disable auto L0 -> L1 compaction + setLevelZeroFileNumCompactionTrigger(20). + setTargetFileSizeBase(L0_FILE_SIZE * 100). setTargetFileSizeMultiplier(1). - setMaxBytesForLevelBase(500 << 10). - setMaxBytesForLevelMultiplier(1). - setDisableAutoCompactions(false))); + // To disable auto compaction + setMaxBytesForLevelBase(NUM_L0_FILES * L0_FILE_SIZE * 100). + setMaxBytesForLevelMultiplier(2). + setDisableAutoCompactions(true))); // open database db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath(), columnFamilyDescriptors, columnFamilyHandles); // fill database with key/value pairs - byte[] b = new byte[10000]; - for (int i = 0; i < 200; i++) { - rand.nextBytes(b); - db.put(columnFamilyHandles.get(1), - String.valueOf(i).getBytes(), b); - } - db.flush(new FlushOptions().setWaitForFlush(true), - columnFamilyHandles.get(1)); - // free column families - for (ColumnFamilyHandle handle : columnFamilyHandles) { - handle.dispose(); - } - // clear column family handles for reopen - columnFamilyHandles.clear(); - db.close(); - columnFamilyDescriptors.get(1). - columnFamilyOptions(). - setTargetFileSizeBase(Long.MAX_VALUE). - setTargetFileSizeMultiplier(1). - setMaxBytesForLevelBase(Long.MAX_VALUE). - setMaxBytesForLevelMultiplier(1). - setDisableAutoCompactions(true); - // reopen database - db = RocksDB.open(opt, - dbFolder.getRoot().getAbsolutePath(), - columnFamilyDescriptors, - columnFamilyHandles); - // compact new column family - db.compactRange(columnFamilyHandles.get(1), true, 0, 0); - // check if new column family is compacted to level zero - for (int i = 0; i < 4; i++) { - if (i == 0) { - assertThat(db.getProperty(columnFamilyHandles.get(1), - "rocksdb.num-files-at-level" + i)). - isEqualTo("1"); - } else { - assertThat(db.getProperty(columnFamilyHandles.get(1), - "rocksdb.num-files-at-level" + i)). - isEqualTo("0"); + byte[] value = new byte[VALUE_SIZE]; + int int_key = 0; + for (int round = 0; round < 5; ++round) { + int initial_key = int_key; + for (int f = 1; f <= NUM_L0_FILES; ++f) { + for (int i = 0; i < NUM_KEYS_PER_L0_FILE; ++i) { + int_key += KEY_INTERVAL; + rand.nextBytes(value); + + db.put(columnFamilyHandles.get(1), + String.format("%020d", int_key).getBytes(), + value); + } + db.flush(new FlushOptions().setWaitForFlush(true), + columnFamilyHandles.get(1)); + // Make sure we do create one more L0 files. + assertThat( + db.getProperty(columnFamilyHandles.get(1), + "rocksdb.num-files-at-level0")). + isEqualTo("" + f); } + + // Compact all L0 files we just created + db.compactRange( + columnFamilyHandles.get(1), + String.format("%020d", initial_key).getBytes(), + String.format("%020d", int_key - 1).getBytes()); + // Making sure there isn't any L0 files. + assertThat( + db.getProperty(columnFamilyHandles.get(1), + "rocksdb.num-files-at-level0")). + isEqualTo("0"); + // Making sure there are some L1 files. + // Here we only use != 0 instead of a specific number + // as we don't want the test make any assumption on + // how compaction works. + assertThat( + db.getProperty(columnFamilyHandles.get(1), + "rocksdb.num-files-at-level1")). + isNotEqualTo("0"); + // Because we only compacted those keys we issued + // in this round, there shouldn't be any L1 -> L2 + // compaction. So we expect zero L2 files here. + assertThat( + db.getProperty(columnFamilyHandles.get(1), + "rocksdb.num-files-at-level2")). + isEqualTo("0"); } } finally { for (ColumnFamilyHandle handle : columnFamilyHandles) {