From 72afdf20fc3d25ea10adf620831e867f68b74075 Mon Sep 17 00:00:00 2001 From: Ben Clay Date: Tue, 30 Oct 2018 13:08:26 -0700 Subject: [PATCH] RocksJava: Add more flags to BlockBasedTableConfig (#4589) Summary: Punch through more flags for BlockBasedTableConfig, mostly around caching index + filter blocks and partitioned filters. sagar0 adamretter Pull Request resolved: https://github.com/facebook/rocksdb/pull/4589 Differential Revision: D12840626 Pulled By: sagar0 fbshipit-source-id: 3c289d367ceb2a012023aa791b990a437dd1393a --- java/rocksjni/table.cc | 10 +- .../org/rocksdb/BlockBasedTableConfig.java | 112 +++++++++++++++++- .../rocksdb/BlockBasedTableConfigTest.java | 40 +++++++ 3 files changed, 155 insertions(+), 7 deletions(-) diff --git a/java/rocksjni/table.cc b/java/rocksjni/table.cc index 5f5f8cd2a..3dbd13280 100644 --- a/java/rocksjni/table.cc +++ b/java/rocksjni/table.cc @@ -37,7 +37,7 @@ jlong Java_org_rocksdb_PlainTableConfig_newTableFactoryHandle( /* * Class: org_rocksdb_BlockBasedTableConfig * Method: newTableFactoryHandle - * Signature: (ZJIJJIIZIZZZJIBBI)J + * Signature: (ZJIJJIIZJZZZZJZZJIBBI)J */ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle( JNIEnv * /*env*/, jobject /*jobj*/, jboolean no_block_cache, @@ -45,7 +45,10 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle( jlong block_size, jint block_size_deviation, jint block_restart_interval, jboolean whole_key_filtering, jlong jfilter_policy, jboolean cache_index_and_filter_blocks, + jboolean cache_index_and_filter_blocks_with_high_priority, jboolean pin_l0_filter_and_index_blocks_in_cache, + jboolean partition_filters, jlong metadata_block_size, + jboolean pin_top_level_index_and_filter, jboolean hash_index_allow_collision, jlong block_cache_compressed_size, jint block_cache_compressd_num_shard_bits, jbyte jchecksum_type, jbyte jindex_type, jint jformat_version) { @@ -77,8 +80,13 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle( options.filter_policy = *pFilterPolicy; } options.cache_index_and_filter_blocks = cache_index_and_filter_blocks; + options.cache_index_and_filter_blocks_with_high_priority = + cache_index_and_filter_blocks_with_high_priority; options.pin_l0_filter_and_index_blocks_in_cache = pin_l0_filter_and_index_blocks_in_cache; + options.partition_filters = partition_filters; + options.metadata_block_size = metadata_block_size; + options.pin_top_level_index_and_filter = pin_top_level_index_and_filter; options.hash_index_allow_collision = hash_index_allow_collision; if (block_cache_compressed_size > 0) { if (block_cache_compressd_num_shard_bits > 0) { diff --git a/java/src/main/java/org/rocksdb/BlockBasedTableConfig.java b/java/src/main/java/org/rocksdb/BlockBasedTableConfig.java index 2dbbc64d3..1032be6e7 100644 --- a/java/src/main/java/org/rocksdb/BlockBasedTableConfig.java +++ b/java/src/main/java/org/rocksdb/BlockBasedTableConfig.java @@ -22,7 +22,11 @@ public class BlockBasedTableConfig extends TableFormatConfig { wholeKeyFiltering_ = true; filter_ = null; cacheIndexAndFilterBlocks_ = false; + cacheIndexAndFilterBlocksWithHighPriority_ = false; pinL0FilterAndIndexBlocksInCache_ = false; + partitionFilters_ = false; + metadataBlockSize_ = 4096; + pinTopLevelIndexAndFilter_ = true; hashIndexAllowCollision_ = true; blockCacheCompressedSize_ = 0; blockCacheCompressedNumShardBits_ = 0; @@ -246,6 +250,31 @@ public class BlockBasedTableConfig extends TableFormatConfig { return this; } + /** + * Indicates if index and filter blocks will be treated as high-priority in the block cache. + * See note below about applicability. If not specified, defaults to false. + * + * @return if index and filter blocks will be treated as high-priority. + */ + public boolean cacheIndexAndFilterBlocksWithHighPriority() { + return cacheIndexAndFilterBlocksWithHighPriority_; + } + + /** + * If true, cache index and filter blocks with high priority. If set to true, + * depending on implementation of block cache, index and filter blocks may be + * less likely to be evicted than data blocks. + * + * @param cacheIndexAndFilterBlocksWithHighPriority if index and filter blocks + * will be treated as high-priority. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setCacheIndexAndFilterBlocksWithHighPriority( + final boolean cacheIndexAndFilterBlocksWithHighPriority) { + cacheIndexAndFilterBlocksWithHighPriority_ = cacheIndexAndFilterBlocksWithHighPriority; + return this; + } + /** * Indicating if we'd like to pin L0 index/filter blocks to the block cache. If not specified, defaults to false. @@ -269,6 +298,70 @@ public class BlockBasedTableConfig extends TableFormatConfig { return this; } + /** + * Indicating if we're using partitioned filters. Defaults to false. + * + * @return if we're using partition filters. + */ + public boolean partitionFilters() { + return partitionFilters_; + } + + /** + * Use partitioned full filters for each SST file. This option is incompatible with + * block-based filters. + * + * @param partitionFilters use partition filters. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setPartitionFilters(final boolean partitionFilters) { + partitionFilters_ = partitionFilters; + return this; + } + + /** + * @return block size for partitioned metadata. + */ + public long metadataBlockSize() { + return metadataBlockSize_; + } + + /** + * Set block size for partitioned metadata. + * + * @param metadataBlockSize Partitioned metadata block size. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setMetadataBlockSize( + final long metadataBlockSize) { + metadataBlockSize_ = metadataBlockSize; + return this; + } + + /** + * Indicates if top-level index and filter blocks should be pinned. + * + * @return if top-level index and filter blocks should be pinned. + */ + public boolean pinTopLevelIndexAndFilter() { + return pinTopLevelIndexAndFilter_; + } + + /** + * If cacheIndexAndFilterBlocks is true and the below is true, then + * the top-level index of partitioned filter and index blocks are stored in + * the cache, but a reference is held in the "table reader" object so the + * blocks are pinned and only evicted from cache when the table reader is + * freed. This is not limited to l0 in LSM tree. + * + * @param pinTopLevelIndexAndFilter if top-level index and filter blocks should be pinned. + * @return the reference to the current config. + */ + public BlockBasedTableConfig setPinTopLevelIndexAndFilter(final boolean pinTopLevelIndexAndFilter) { + pinTopLevelIndexAndFilter_ = pinTopLevelIndexAndFilter; + return this; + } + /** * Influence the behavior when kHashSearch is used. if false, stores a precise prefix to block range mapping @@ -440,20 +533,27 @@ public class BlockBasedTableConfig extends TableFormatConfig { return newTableFactoryHandle(noBlockCache_, blockCacheSize_, blockCacheNumShardBits_, blockCacheHandle, blockSize_, blockSizeDeviation_, blockRestartInterval_, wholeKeyFiltering_, filterHandle, cacheIndexAndFilterBlocks_, - pinL0FilterAndIndexBlocksInCache_, hashIndexAllowCollision_, blockCacheCompressedSize_, - blockCacheCompressedNumShardBits_, checksumType_.getValue(), indexType_.getValue(), - formatVersion_); + cacheIndexAndFilterBlocksWithHighPriority_, pinL0FilterAndIndexBlocksInCache_, + partitionFilters_, metadataBlockSize_, pinTopLevelIndexAndFilter_, + hashIndexAllowCollision_, blockCacheCompressedSize_, blockCacheCompressedNumShardBits_, + checksumType_.getValue(), indexType_.getValue(), formatVersion_); } private native long newTableFactoryHandle(boolean noBlockCache, long blockCacheSize, int blockCacheNumShardBits, long blockCacheHandle, long blockSize, int blockSizeDeviation, int blockRestartInterval, boolean wholeKeyFiltering, long filterPolicyHandle, - boolean cacheIndexAndFilterBlocks, boolean pinL0FilterAndIndexBlocksInCache, - boolean hashIndexAllowCollision, long blockCacheCompressedSize, - int blockCacheCompressedNumShardBits, byte checkSumType, byte indexType, int formatVersion); + boolean cacheIndexAndFilterBlocks, boolean cacheIndexAndFilterBlocksWithHighPriority, + boolean pinL0FilterAndIndexBlocksInCache, boolean partitionFilters, long metadataBlockSize, + boolean pinTopLevelIndexAndFilter, boolean hashIndexAllowCollision, + long blockCacheCompressedSize, int blockCacheCompressedNumShardBits, + byte checkSumType, byte indexType, int formatVersion); private boolean cacheIndexAndFilterBlocks_; + private boolean cacheIndexAndFilterBlocksWithHighPriority_; private boolean pinL0FilterAndIndexBlocksInCache_; + private boolean partitionFilters_; + private long metadataBlockSize_; + private boolean pinTopLevelIndexAndFilter_; private IndexType indexType_; private boolean hashIndexAllowCollision_; private ChecksumType checksumType_; diff --git a/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java b/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java index 2b15b69f8..754cf11c0 100644 --- a/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java +++ b/java/src/test/java/org/rocksdb/BlockBasedTableConfigTest.java @@ -95,6 +95,46 @@ public class BlockBasedTableConfigTest { } + @Test + public void cacheIndexAndFilterBlocksWithHighPriority() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setCacheIndexAndFilterBlocksWithHighPriority(true); + assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocksWithHighPriority()). + isTrue(); + } + + @Test + public void pinL0FilterAndIndexBlocksInCache() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setPinL0FilterAndIndexBlocksInCache(true); + assertThat(blockBasedTableConfig.pinL0FilterAndIndexBlocksInCache()). + isTrue(); + } + + @Test + public void partitionFilters() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setPartitionFilters(true); + assertThat(blockBasedTableConfig.partitionFilters()). + isTrue(); + } + + @Test + public void metadataBlockSize() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setMetadataBlockSize(1024); + assertThat(blockBasedTableConfig.metadataBlockSize()). + isEqualTo(1024); + } + + @Test + public void pinTopLevelIndexAndFilter() { + BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(); + blockBasedTableConfig.setPinTopLevelIndexAndFilter(false); + assertThat(blockBasedTableConfig.pinTopLevelIndexAndFilter()). + isFalse(); + } + @Test public void hashIndexAllowCollision() { BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();