NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

memtable_prefix_bloom_bits -> memtable_prefix_bloom_bits_ratio and deprecate memtable_prefix_bloom_probes

Browse Source 
Summary:
memtable_prefix_bloom_probes is not a critical option. Remove it to reduce number of options.
It's easier for users to make mistakes with memtable_prefix_bloom_bits, turn it to memtable_prefix_bloom_bits_ratio

Test Plan: Run all existing tests

Reviewers: yhchiang, igor, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: gunnarku, yoshinorim, MarkCallaghan, leveldb, andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D59199
main
sdong 9 years ago
parent e9c1face6c
commit 20699df843
24 changed files with 127 additions and 249 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      HISTORY.md
  2. 11
      db/c.cc
  3. 6
      db/column_family.cc
  4. 3
      db/db_bloom_filter_test.cc
  5. 45
      db/memtable.cc
  6. 1
      db/memtable.h
  7. 7
      db/prefix_test.cc
  8. 1
      db/version_set.h
  9. 13
      include/rocksdb/options.h
  10. 91
      java/rocksjni/options.cc
  11. 30
      java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
  12. 25
      java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java
  13. 30
      java/src/main/java/org/rocksdb/Options.java
  14. 17
      java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
  15. 17
      java/src/test/java/org/rocksdb/OptionsTest.java
  16. 7
      tools/db_bench_tool.cc
  17. 6
      util/mutable_cf_options.cc
  18. 10
      util/mutable_cf_options.h
  19. 13
      util/options.cc
  20. 12
      util/options_helper.cc
  21. 9
      util/options_helper.h
  22. 3
      util/options_settable_test.cc
  23. 12
      util/options_test.cc
  24. 4
      util/testutil.cc

3
HISTORY.md
Unescape View File

@ -1,5 +1,8 @@
# Rocksdb Change Log # Rocksdb Change Log
## Unreleased ## Unreleased
### Public API Change
* Deprecate BlockBaseTableOptions.hash_index_allow_collision=false
* options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes
## 4.9.0 (6/9/2016) ## 4.9.0 (6/9/2016)
### Public API changes ### Public API changes

11
db/c.cc
Unescape View File

@ -1799,14 +1799,9 @@ void rocksdb_options_set_memtable_vector_rep(rocksdb_options_t *opt) {
opt->rep.memtable_factory.reset(factory); opt->rep.memtable_factory.reset(factory);
} }
void rocksdb_options_set_memtable_prefix_bloom_bits( void rocksdb_options_set_memtable_prefix_bloom_size_ratio(
rocksdb_options_t* opt, uint32_t v) { rocksdb_options_t* opt, double v) {
opt->rep.memtable_prefix_bloom_bits = v; opt->rep.memtable_prefix_bloom_size_ratio = v;
}
void rocksdb_options_set_memtable_prefix_bloom_probes(
rocksdb_options_t* opt, uint32_t v) {
opt->rep.memtable_prefix_bloom_probes = v;
} }
void rocksdb_options_set_memtable_prefix_bloom_huge_page_tlb_size( void rocksdb_options_set_memtable_prefix_bloom_huge_page_tlb_size(

6
db/column_family.cc
Unescape View File

@ -177,6 +177,12 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
if (result.max_write_buffer_number_to_maintain < 0) { if (result.max_write_buffer_number_to_maintain < 0) {
result.max_write_buffer_number_to_maintain = result.max_write_buffer_number; result.max_write_buffer_number_to_maintain = result.max_write_buffer_number;
} }
// bloom filter size shouldn't exceed 1/4 of memtable size.
if (result.memtable_prefix_bloom_size_ratio > 0.25) {
result.memtable_prefix_bloom_size_ratio = 0.25;
} else if (result.memtable_prefix_bloom_size_ratio < 0) {
result.memtable_prefix_bloom_size_ratio = 0;
}
XFUNC_TEST("memtablelist_history", "transaction_xftest_SanitizeOptions", XFUNC_TEST("memtablelist_history", "transaction_xftest_SanitizeOptions",
xf_transaction_set_memtable_history1, xf_transaction_set_memtable_history1,
xf_transaction_set_memtable_history, xf_transaction_set_memtable_history,

3
db/db_bloom_filter_test.cc
Unescape View File

@ -615,7 +615,8 @@ class BloomStatsTestWithParam
options_.create_if_missing = true; options_.create_if_missing = true;
options_.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(4)); options_.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(4));
options_.memtable_prefix_bloom_bits = 8 * 1024; options_.memtable_prefix_bloom_size_ratio =
8.0 * 1024.0 / static_cast<double>(options_.write_buffer_size);
if (use_block_table_) { if (use_block_table_) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.hash_index_allow_collision = false; table_options.hash_index_allow_collision = false;

45
db/memtable.cc
Unescape View File

@ -34,24 +34,25 @@
namespace rocksdb { namespace rocksdb {
MemTableOptions::MemTableOptions( MemTableOptions::MemTableOptions(const ImmutableCFOptions& ioptions,
const ImmutableCFOptions& ioptions, const MutableCFOptions& mutable_cf_options)
const MutableCFOptions& mutable_cf_options) : write_buffer_size(mutable_cf_options.write_buffer_size),
: write_buffer_size(mutable_cf_options.write_buffer_size), arena_block_size(mutable_cf_options.arena_block_size),
arena_block_size(mutable_cf_options.arena_block_size), memtable_prefix_bloom_bits(
memtable_prefix_bloom_bits(mutable_cf_options.memtable_prefix_bloom_bits), static_cast<uint32_t>(
memtable_prefix_bloom_probes( static_cast<double>(mutable_cf_options.write_buffer_size) *
mutable_cf_options.memtable_prefix_bloom_probes), mutable_cf_options.memtable_prefix_bloom_size_ratio) *
memtable_prefix_bloom_huge_page_tlb_size( 8u),
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size), memtable_prefix_bloom_huge_page_tlb_size(
inplace_update_support(ioptions.inplace_update_support), mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size),
inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks), inplace_update_support(ioptions.inplace_update_support),
inplace_callback(ioptions.inplace_callback), inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks),
max_successive_merges(mutable_cf_options.max_successive_merges), inplace_callback(ioptions.inplace_callback),
filter_deletes(mutable_cf_options.filter_deletes), max_successive_merges(mutable_cf_options.max_successive_merges),
statistics(ioptions.statistics), filter_deletes(mutable_cf_options.filter_deletes),
merge_operator(ioptions.merge_operator), statistics(ioptions.statistics),
info_log(ioptions.info_log) {} merge_operator(ioptions.merge_operator),
info_log(ioptions.info_log) {}
MemTable::MemTable(const InternalKeyComparator& cmp, MemTable::MemTable(const InternalKeyComparator& cmp,
const ImmutableCFOptions& ioptions, const ImmutableCFOptions& ioptions,
@ -88,11 +89,9 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) { if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) {
prefix_bloom_.reset(new DynamicBloom( prefix_bloom_.reset(new DynamicBloom(
&allocator_, &allocator_, moptions_.memtable_prefix_bloom_bits,
moptions_.memtable_prefix_bloom_bits, ioptions.bloom_locality, ioptions.bloom_locality, 6 /* hard coded 6 probes */, nullptr,
moptions_.memtable_prefix_bloom_probes, nullptr, moptions_.memtable_prefix_bloom_huge_page_tlb_size, ioptions.info_log));
moptions_.memtable_prefix_bloom_huge_page_tlb_size,
ioptions.info_log));
} }
} }

1
db/memtable.h
Unescape View File

@ -42,7 +42,6 @@ struct MemTableOptions {
size_t write_buffer_size; size_t write_buffer_size;
size_t arena_block_size; size_t arena_block_size;
uint32_t memtable_prefix_bloom_bits; uint32_t memtable_prefix_bloom_bits;
uint32_t memtable_prefix_bloom_probes;
size_t memtable_prefix_bloom_huge_page_tlb_size; size_t memtable_prefix_bloom_huge_page_tlb_size;
bool inplace_update_support; bool inplace_update_support;
size_t inplace_update_num_locks; size_t inplace_update_num_locks;

7
db/prefix_test.cc
Unescape View File

@ -43,8 +43,7 @@ DEFINE_int64(write_buffer_size, 33554432, "");
DEFINE_int32(max_write_buffer_number, 2, ""); DEFINE_int32(max_write_buffer_number, 2, "");
DEFINE_int32(min_write_buffer_number_to_merge, 1, ""); DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
DEFINE_int32(skiplist_height, 4, ""); DEFINE_int32(skiplist_height, 4, "");
DEFINE_int32(memtable_prefix_bloom_bits, 10000000, ""); DEFINE_double(memtable_prefix_bloom_size_ratio, 0.1, "");
DEFINE_int32(memtable_prefix_bloom_probes, 10, "");
DEFINE_int32(memtable_prefix_bloom_huge_page_tlb_size, 2 * 1024 * 1024, ""); DEFINE_int32(memtable_prefix_bloom_huge_page_tlb_size, 2 * 1024 * 1024, "");
DEFINE_int32(value_size, 40, ""); DEFINE_int32(value_size, 40, "");
@ -160,8 +159,8 @@ class PrefixTest : public testing::Test {
options.min_write_buffer_number_to_merge = options.min_write_buffer_number_to_merge =
FLAGS_min_write_buffer_number_to_merge; FLAGS_min_write_buffer_number_to_merge;
options.memtable_prefix_bloom_bits = FLAGS_memtable_prefix_bloom_bits; options.memtable_prefix_bloom_size_ratio =
options.memtable_prefix_bloom_probes = FLAGS_memtable_prefix_bloom_probes; FLAGS_memtable_prefix_bloom_size_ratio;
options.memtable_prefix_bloom_huge_page_tlb_size = options.memtable_prefix_bloom_huge_page_tlb_size =
FLAGS_memtable_prefix_bloom_huge_page_tlb_size; FLAGS_memtable_prefix_bloom_huge_page_tlb_size;

1
db/version_set.h
Unescape View File

@ -57,7 +57,6 @@ class Version;
class VersionSet; class VersionSet;
class WriteBuffer; class WriteBuffer;
class MergeContext; class MergeContext;
class ColumnFamilyData;
class ColumnFamilySet; class ColumnFamilySet;
class TableCache; class TableCache;
class MergeIteratorBuilder; class MergeIteratorBuilder;

13
include/rocksdb/options.h
Unescape View File

@ -745,16 +745,15 @@ struct ColumnFamilyOptions {
Slice delta_value, Slice delta_value,
std::string* merged_value); std::string* merged_value);
// if prefix_extractor is set and bloom_bits is not 0, create prefix bloom // if prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
// for memtable // create prefix bloom for memtable with the size of
// write_buffer_size * memtable_prefix_bloom_size_ratio.
// If it is larger than 0.25, it is santinized to 0.25.
// //
// Dynamically changeable through SetOptions() API // Default: 0 (disable)
uint32_t memtable_prefix_bloom_bits;
// number of hash probes per key
// //
// Dynamically changeable through SetOptions() API // Dynamically changeable through SetOptions() API
uint32_t memtable_prefix_bloom_probes; double memtable_prefix_bloom_size_ratio;
// Page size for huge page TLB for bloom in memtable. If <=0, not allocate // Page size for huge page TLB for bloom in memtable. If <=0, not allocate
// from huge page TLB but from malloc. // from huge page TLB but from malloc.

91
java/rocksjni/options.cc
Unescape View File

@ -1732,50 +1732,27 @@ void Java_org_rocksdb_Options_setInplaceUpdateNumLocks(
/* /*
* Class: org_rocksdb_Options * Class: org_rocksdb_Options
* Method: memtablePrefixBloomBits * Method: memtablePrefixBloomSizeRatio
* Signature: (J)I * Signature: (J)I
*/ */
jint Java_org_rocksdb_Options_memtablePrefixBloomBits( jdouble Java_org_rocksdb_Options_memtablePrefixBloomSizeRatio(JNIEnv* env,
JNIEnv* env, jobject jobj, jlong jhandle) { jobject jobj,
return reinterpret_cast<rocksdb::Options*>( jlong jhandle) {
jhandle)->memtable_prefix_bloom_bits; return reinterpret_cast<rocksdb::Options*>(jhandle)
} ->memtable_prefix_bloom_size_ratio;
/*
* Class: org_rocksdb_Options
* Method: setMemtablePrefixBloomBits
* Signature: (JI)V
*/
void Java_org_rocksdb_Options_setMemtablePrefixBloomBits(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_bits) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_bits =
static_cast<int32_t>(jmemtable_prefix_bloom_bits);
}
/*
* Class: org_rocksdb_Options
* Method: memtablePrefixBloomProbes
* Signature: (J)I
*/
jint Java_org_rocksdb_Options_memtablePrefixBloomProbes(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_probes;
} }
/* /*
* Class: org_rocksdb_Options * Class: org_rocksdb_Options
* Method: setMemtablePrefixBloomProbes * Method: setMemtablePrefixBloomSizeRatio
* Signature: (JI)V * Signature: (JI)V
*/ */
void Java_org_rocksdb_Options_setMemtablePrefixBloomProbes( void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio(
JNIEnv* env, jobject jobj, jlong jhandle, JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_probes) { jdouble jmemtable_prefix_bloom_size_ratio) {
reinterpret_cast<rocksdb::Options*>( reinterpret_cast<rocksdb::Options*>(jhandle)
jhandle)->memtable_prefix_bloom_probes = ->memtable_prefix_bloom_size_ratio =
static_cast<int32_t>(jmemtable_prefix_bloom_probes); static_cast<double>(jmemtable_prefix_bloom_size_ratio);
} }
/* /*
@ -2915,50 +2892,26 @@ void Java_org_rocksdb_ColumnFamilyOptions_setInplaceUpdateNumLocks(
/* /*
* Class: org_rocksdb_ColumnFamilyOptions * Class: org_rocksdb_ColumnFamilyOptions
* Method: memtablePrefixBloomBits * Method: memtablePrefixBloomSizeRatio
* Signature: (J)I * Signature: (J)I
*/ */
jint Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomBits( jdouble Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomSizeRatio(
JNIEnv* env, jobject jobj, jlong jhandle) { JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>( return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)
jhandle)->memtable_prefix_bloom_bits; ->memtable_prefix_bloom_size_ratio;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMemtablePrefixBloomBits
* Signature: (JI)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomBits(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_bits) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_bits =
static_cast<int32_t>(jmemtable_prefix_bloom_bits);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: memtablePrefixBloomProbes
* Signature: (J)I
*/
jint Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomProbes(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_probes;
} }
/* /*
* Class: org_rocksdb_ColumnFamilyOptions * Class: org_rocksdb_ColumnFamilyOptions
* Method: setMemtablePrefixBloomProbes * Method: setMemtablePrefixBloomSizeRatio
* Signature: (JI)V * Signature: (JI)V
*/ */
void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomProbes( void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio(
JNIEnv* env, jobject jobj, jlong jhandle, JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_probes) { jdouble jmemtable_prefix_bloom_size_ratio) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>( reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)
jhandle)->memtable_prefix_bloom_probes = ->memtable_prefix_bloom_size_ratio =
static_cast<int32_t>(jmemtable_prefix_bloom_probes); static_cast<double>(jmemtable_prefix_bloom_size_ratio);
} }
/* /*

30
java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
Unescape View File

@ -589,27 +589,15 @@ public class ColumnFamilyOptions extends RocksObject
} }
@Override @Override
public ColumnFamilyOptions setMemtablePrefixBloomBits( public ColumnFamilyOptions setMemtablePrefixBloomSizeRatio(
final int memtablePrefixBloomBits) { final double memtablePrefixBloomSizeRatio) {
setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomBits); setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomSizeRatio);
return this; return this;
} }
@Override @Override
public int memtablePrefixBloomBits() { public double memtablePrefixBloomSizeRatio() {
return memtablePrefixBloomBits(nativeHandle_); return memtablePrefixBloomSizeRatio(nativeHandle_);
}
@Override
public ColumnFamilyOptions setMemtablePrefixBloomProbes(
final int memtablePrefixBloomProbes) {
setMemtablePrefixBloomProbes(nativeHandle_, memtablePrefixBloomProbes);
return this;
}
@Override
public int memtablePrefixBloomProbes() {
return memtablePrefixBloomProbes(nativeHandle_);
} }
@Override @Override
@ -787,12 +775,8 @@ public class ColumnFamilyOptions extends RocksObject
long handle, long inplaceUpdateNumLocks) long handle, long inplaceUpdateNumLocks)
throws IllegalArgumentException; throws IllegalArgumentException;
private native long inplaceUpdateNumLocks(long handle); private native long inplaceUpdateNumLocks(long handle);
private native void setMemtablePrefixBloomBits( private native void setMemtablePrefixBloomBits(long handle, double memtablePrefixBloomSizeRatio);
long handle, int memtablePrefixBloomBits); private native double memtablePrefixBloomSizeRatio(long handle);
private native int memtablePrefixBloomBits(long handle);
private native void setMemtablePrefixBloomProbes(
long handle, int memtablePrefixBloomProbes);
private native int memtablePrefixBloomProbes(long handle);
private native void setBloomLocality( private native void setBloomLocality(
long handle, int bloomLocality); long handle, int bloomLocality);
private native int bloomLocality(long handle); private native int bloomLocality(long handle);

25
java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java
Unescape View File

@ -1011,15 +1011,15 @@ public interface ColumnFamilyOptionsInterface {
long inplaceUpdateNumLocks(); long inplaceUpdateNumLocks();
/** /**
* Sets the number of bits used in the prefix bloom filter. * Sets the size ratio of the memtable used in the prefix bloom filter.
* *
* This value will be used only when a prefix-extractor is specified. * This value will be used only when a prefix-extractor is specified.
* *
* @param memtablePrefixBloomBits the number of bits used in the * @param memtablePrefixBloomSizeRatio the number of bits used in the
* prefix bloom filter. * prefix bloom filter.
* @return the reference to the current option. * @return the reference to the current option.
*/ */
Object setMemtablePrefixBloomBits(int memtablePrefixBloomBits); Object setMemtablePrefixBloomSizeRatio(double memtablePrefixBloomSizeRatio);
/** /**
* Returns the number of bits used in the prefix bloom filter. * Returns the number of bits used in the prefix bloom filter.
@ -1029,22 +1029,7 @@ public interface ColumnFamilyOptionsInterface {
* @return the number of bloom-bits. * @return the number of bloom-bits.
* @see #useFixedLengthPrefixExtractor(int) * @see #useFixedLengthPrefixExtractor(int)
*/ */
int memtablePrefixBloomBits(); double memtablePrefixBloomSizeRatio();
/**
* The number of hash probes per key used in the mem-table.
*
* @param memtablePrefixBloomProbes the number of hash probes per key.
* @return the reference to the current option.
*/
Object setMemtablePrefixBloomProbes(int memtablePrefixBloomProbes);
/**
* The number of hash probes per key used in the mem-table.
*
* @return the number of hash probes per key.
*/
int memtablePrefixBloomProbes();
/** /**
* Control locality of bloom filter probes to improve cache miss rate. * Control locality of bloom filter probes to improve cache miss rate.
@ -1074,7 +1059,7 @@ public interface ColumnFamilyOptionsInterface {
* Default: 0 * Default: 0
* *
* @return the level of locality of bloom-filter probes. * @return the level of locality of bloom-filter probes.
* @see #setMemtablePrefixBloomProbes(int) * @see #setBloomLocality(int)
*/ */
int bloomLocality(); int bloomLocality();

30
java/src/main/java/org/rocksdb/Options.java
Unescape View File

@ -1005,26 +1005,13 @@ public class Options extends RocksObject
} }
@Override @Override
public int memtablePrefixBloomBits() { public double memtablePrefixBloomSizeRatio() {
return memtablePrefixBloomBits(nativeHandle_); return memtablePrefixBloomSizeRatio(nativeHandle_);
} }
@Override @Override
public Options setMemtablePrefixBloomBits( public Options setMemtablePrefixBloomSizeRatio(final double memtablePrefixBloomSizeRatio) {
final int memtablePrefixBloomBits) { setMemtablePrefixBloomSizeRatio(nativeHandle_, memtablePrefixBloomSizeRatio);
setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomBits);
return this;
}
@Override
public int memtablePrefixBloomProbes() {
return memtablePrefixBloomProbes(nativeHandle_);
}
@Override
public Options setMemtablePrefixBloomProbes(
final int memtablePrefixBloomProbes) {
setMemtablePrefixBloomProbes(nativeHandle_, memtablePrefixBloomProbes);
return this; return this;
} }
@ -1293,12 +1280,9 @@ public class Options extends RocksObject
long handle, long inplaceUpdateNumLocks) long handle, long inplaceUpdateNumLocks)
throws IllegalArgumentException; throws IllegalArgumentException;
private native long inplaceUpdateNumLocks(long handle); private native long inplaceUpdateNumLocks(long handle);
private native void setMemtablePrefixBloomBits( private native void setMemtablePrefixBloomSizeRatio(
long handle, int memtablePrefixBloomBits); long handle, double memtablePrefixBloomSizeRatio);
private native int memtablePrefixBloomBits(long handle); private native double memtablePrefixBloomSizeRatio(long handle);
private native void setMemtablePrefixBloomProbes(
long handle, int memtablePrefixBloomProbes);
private native int memtablePrefixBloomProbes(long handle);
private native void setBloomLocality( private native void setBloomLocality(
long handle, int bloomLocality); long handle, int bloomLocality);
private native int bloomLocality(long handle); private native int bloomLocality(long handle);

17
java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
Unescape View File

@ -305,20 +305,11 @@ public class ColumnFamilyOptionsTest {
} }
@Test @Test
public void memtablePrefixBloomBits() { public void memtablePrefixBloomSizeRatio() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) { try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final int intValue = rand.nextInt(); final double doubleValue = rand.nextDouble();
opt.setMemtablePrefixBloomBits(intValue); opt.setMemtablePrefixBloomSizeRatio(doubleValue);
assertThat(opt.memtablePrefixBloomBits()).isEqualTo(intValue); assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue);
}
}
@Test
public void memtablePrefixBloomProbes() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomProbes(intValue);
assertThat(opt.memtablePrefixBloomProbes()).isEqualTo(intValue);
} }
} }

17
java/src/test/java/org/rocksdb/OptionsTest.java
Unescape View File

@ -268,20 +268,11 @@ public class OptionsTest {
} }
@Test @Test
public void memtablePrefixBloomBits() { public void memtablePrefixBloomSizeRatio() {
try (final Options opt = new Options()) { try (final Options opt = new Options()) {
final int intValue = rand.nextInt(); final double doubleValue = rand.nextDouble();
opt.setMemtablePrefixBloomBits(intValue); opt.setMemtablePrefixBloomSizeRatio(doubleValue);
assertThat(opt.memtablePrefixBloomBits()).isEqualTo(intValue); assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue);
}
}
@Test
public void memtablePrefixBloomProbes() {
try (final Options opt = new Options()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomProbes(intValue);
assertThat(opt.memtablePrefixBloomProbes()).isEqualTo(intValue);
} }
} }

7
tools/db_bench_tool.cc
Unescape View File

@ -396,8 +396,9 @@ DEFINE_int32(skip_table_builder_flush, false, "Skip flushing block in "
DEFINE_int32(bloom_bits, -1, "Bloom filter bits per key. Negative means" DEFINE_int32(bloom_bits, -1, "Bloom filter bits per key. Negative means"
" use default settings."); " use default settings.");
DEFINE_int32(memtable_bloom_bits, 0, "Bloom filter bits per key for memtable. " DEFINE_double(memtable_bloom_size_ratio, 0,
"Negative means no bloom filter."); "Ratio of memtable size used for bloom filter. 0 means no bloom "
"filter.");
DEFINE_bool(use_existing_db, false, "If true, do not destroy the existing" DEFINE_bool(use_existing_db, false, "If true, do not destroy the existing"
" database. If you set this flag and also specify a benchmark that" " database. If you set this flag and also specify a benchmark that"
@ -2461,7 +2462,7 @@ class Benchmark {
exit(1); exit(1);
} }
} }
options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits; options.memtable_prefix_bloom_size_ratio = FLAGS_memtable_bloom_size_ratio;
options.bloom_locality = FLAGS_bloom_locality; options.bloom_locality = FLAGS_bloom_locality;
options.max_file_opening_threads = FLAGS_file_opening_threads; options.max_file_opening_threads = FLAGS_file_opening_threads;
options.new_table_reader_for_compaction_inputs = options.new_table_reader_for_compaction_inputs =

6
util/mutable_cf_options.cc
Unescape View File

@ -70,10 +70,8 @@ void MutableCFOptions::Dump(Logger* log) const {
max_write_buffer_number); max_write_buffer_number);
Log(log, " arena_block_size: %" ROCKSDB_PRIszt, Log(log, " arena_block_size: %" ROCKSDB_PRIszt,
arena_block_size); arena_block_size);
Log(log, " memtable_prefix_bloom_bits: %" PRIu32, Log(log, " memtable_prefix_bloom_ratio: %f",
memtable_prefix_bloom_bits); memtable_prefix_bloom_size_ratio);
Log(log, " memtable_prefix_bloom_probes: %" PRIu32,
memtable_prefix_bloom_probes);
Log(log, " memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt, Log(log, " memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt,
memtable_prefix_bloom_huge_page_tlb_size); memtable_prefix_bloom_huge_page_tlb_size);
Log(log, " max_successive_merges: %" ROCKSDB_PRIszt, Log(log, " max_successive_merges: %" ROCKSDB_PRIszt,

10
util/mutable_cf_options.h
Unescape View File

@ -17,8 +17,8 @@ struct MutableCFOptions {
: write_buffer_size(options.write_buffer_size), : write_buffer_size(options.write_buffer_size),
max_write_buffer_number(options.max_write_buffer_number), max_write_buffer_number(options.max_write_buffer_number),
arena_block_size(options.arena_block_size), arena_block_size(options.arena_block_size),
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits), memtable_prefix_bloom_size_ratio(
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes), options.memtable_prefix_bloom_size_ratio),
memtable_prefix_bloom_huge_page_tlb_size( memtable_prefix_bloom_huge_page_tlb_size(
options.memtable_prefix_bloom_huge_page_tlb_size), options.memtable_prefix_bloom_huge_page_tlb_size),
max_successive_merges(options.max_successive_merges), max_successive_merges(options.max_successive_merges),
@ -58,8 +58,7 @@ struct MutableCFOptions {
: write_buffer_size(0), : write_buffer_size(0),
max_write_buffer_number(0), max_write_buffer_number(0),
arena_block_size(0), arena_block_size(0),
memtable_prefix_bloom_bits(0), memtable_prefix_bloom_size_ratio(0),
memtable_prefix_bloom_probes(0),
memtable_prefix_bloom_huge_page_tlb_size(0), memtable_prefix_bloom_huge_page_tlb_size(0),
max_successive_merges(0), max_successive_merges(0),
filter_deletes(false), filter_deletes(false),
@ -110,8 +109,7 @@ struct MutableCFOptions {
size_t write_buffer_size; size_t write_buffer_size;
int max_write_buffer_number; int max_write_buffer_number;
size_t arena_block_size; size_t arena_block_size;
uint32_t memtable_prefix_bloom_bits; double memtable_prefix_bloom_size_ratio;
uint32_t memtable_prefix_bloom_probes;
size_t memtable_prefix_bloom_huge_page_tlb_size; size_t memtable_prefix_bloom_huge_page_tlb_size;
size_t max_successive_merges; size_t max_successive_merges;
bool filter_deletes; bool filter_deletes;

13
util/options.cc
Unescape View File

@ -121,8 +121,7 @@ ColumnFamilyOptions::ColumnFamilyOptions()
inplace_update_support(false), inplace_update_support(false),
inplace_update_num_locks(10000), inplace_update_num_locks(10000),
inplace_callback(nullptr), inplace_callback(nullptr),
memtable_prefix_bloom_bits(0), memtable_prefix_bloom_size_ratio(0.0),
memtable_prefix_bloom_probes(6),
memtable_prefix_bloom_huge_page_tlb_size(0), memtable_prefix_bloom_huge_page_tlb_size(0),
bloom_locality(0), bloom_locality(0),
max_successive_merges(0), max_successive_merges(0),
@ -190,8 +189,8 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
inplace_update_support(options.inplace_update_support), inplace_update_support(options.inplace_update_support),
inplace_update_num_locks(options.inplace_update_num_locks), inplace_update_num_locks(options.inplace_update_num_locks),
inplace_callback(options.inplace_callback), inplace_callback(options.inplace_callback),
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits), memtable_prefix_bloom_size_ratio(
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes), options.memtable_prefix_bloom_size_ratio),
memtable_prefix_bloom_huge_page_tlb_size( memtable_prefix_bloom_huge_page_tlb_size(
options.memtable_prefix_bloom_huge_page_tlb_size), options.memtable_prefix_bloom_huge_page_tlb_size),
bloom_locality(options.bloom_locality), bloom_locality(options.bloom_locality),
@ -596,10 +595,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
Header(log, " Options.min_partial_merge_operands: %u", Header(log, " Options.min_partial_merge_operands: %u",
min_partial_merge_operands); min_partial_merge_operands);
// TODO: easier config for bloom (maybe based on avg key/value size) // TODO: easier config for bloom (maybe based on avg key/value size)
Header(log, " Options.memtable_prefix_bloom_bits: %d", Header(log, " Options.memtable_prefix_bloom_size_ratio: %f",
memtable_prefix_bloom_bits); memtable_prefix_bloom_size_ratio);
Header(log, " Options.memtable_prefix_bloom_probes: %d",
memtable_prefix_bloom_probes);
Header(log, Header(log,
" Options.memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt, " Options.memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt,

12
util/options_helper.cc
Unescape View File

@ -537,9 +537,11 @@ bool ParseMemtableOptions(const std::string& name, const std::string& value,
} else if (name == "arena_block_size") { } else if (name == "arena_block_size") {
new_options->arena_block_size = ParseSizeT(value); new_options->arena_block_size = ParseSizeT(value);
} else if (name == "memtable_prefix_bloom_bits") { } else if (name == "memtable_prefix_bloom_bits") {
new_options->memtable_prefix_bloom_bits = ParseUint32(value); // deprecated
} else if (name == "memtable_prefix_bloom_size_ratio") {
new_options->memtable_prefix_bloom_size_ratio = ParseDouble(value);
} else if (name == "memtable_prefix_bloom_probes") { } else if (name == "memtable_prefix_bloom_probes") {
new_options->memtable_prefix_bloom_probes = ParseUint32(value); // Deprecated
} else if (name == "memtable_prefix_bloom_huge_page_tlb_size") { } else if (name == "memtable_prefix_bloom_huge_page_tlb_size") {
new_options->memtable_prefix_bloom_huge_page_tlb_size = new_options->memtable_prefix_bloom_huge_page_tlb_size =
ParseSizeT(value); ParseSizeT(value);
@ -1437,10 +1439,8 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size; cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size;
cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number; cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number;
cf_opts.arena_block_size = mutable_cf_options.arena_block_size; cf_opts.arena_block_size = mutable_cf_options.arena_block_size;
cf_opts.memtable_prefix_bloom_bits = cf_opts.memtable_prefix_bloom_size_ratio =
mutable_cf_options.memtable_prefix_bloom_bits; mutable_cf_options.memtable_prefix_bloom_size_ratio;
cf_opts.memtable_prefix_bloom_probes =
mutable_cf_options.memtable_prefix_bloom_probes;
cf_opts.memtable_prefix_bloom_huge_page_tlb_size = cf_opts.memtable_prefix_bloom_huge_page_tlb_size =
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size; mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size;
cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges; cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges;

9
util/options_helper.h
Unescape View File

@ -437,11 +437,12 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{offsetof(struct ColumnFamilyOptions, bloom_locality), {offsetof(struct ColumnFamilyOptions, bloom_locality),
OptionType::kUInt32T, OptionVerificationType::kNormal}}, OptionType::kUInt32T, OptionVerificationType::kNormal}},
{"memtable_prefix_bloom_bits", {"memtable_prefix_bloom_bits",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_bits), {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated}},
OptionType::kUInt32T, OptionVerificationType::kNormal}}, {"memtable_prefix_bloom_size_ratio",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_size_ratio),
OptionType::kDouble, OptionVerificationType::kNormal}},
{"memtable_prefix_bloom_probes", {"memtable_prefix_bloom_probes",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_probes), {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated}},
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{"min_partial_merge_operands", {"min_partial_merge_operands",
{offsetof(struct ColumnFamilyOptions, min_partial_merge_operands), {offsetof(struct ColumnFamilyOptions, min_partial_merge_operands),
OptionType::kUInt32T, OptionVerificationType::kNormal}}, OptionType::kUInt32T, OptionVerificationType::kNormal}},

3
util/options_settable_test.cc
Unescape View File

@ -413,14 +413,13 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"max_write_buffer_number_to_maintain=84;" "max_write_buffer_number_to_maintain=84;"
"verify_checksums_in_compaction=false;" "verify_checksums_in_compaction=false;"
"merge_operator=aabcxehazrMergeOperator;" "merge_operator=aabcxehazrMergeOperator;"
"memtable_prefix_bloom_bits=4642;" "memtable_prefix_bloom_size_ratio=0.4642;"
"paranoid_file_checks=true;" "paranoid_file_checks=true;"
"inplace_update_num_locks=7429;" "inplace_update_num_locks=7429;"
"optimize_filters_for_hits=false;" "optimize_filters_for_hits=false;"
"level_compaction_dynamic_level_bytes=false;" "level_compaction_dynamic_level_bytes=false;"
"inplace_update_support=false;" "inplace_update_support=false;"
"compaction_style=kCompactionStyleFIFO;" "compaction_style=kCompactionStyleFIFO;"
"memtable_prefix_bloom_probes=2511;"
"purge_redundant_kvs_while_flush=true;" "purge_redundant_kvs_while_flush=true;"
"filter_deletes=false;" "filter_deletes=false;"
"hard_pending_compaction_bytes_limit=0;" "hard_pending_compaction_bytes_limit=0;"

12
util/options_test.cc
Unescape View File

@ -86,8 +86,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"report_bg_io_stats", "true"}, {"report_bg_io_stats", "true"},
{"compaction_measure_io_stats", "false"}, {"compaction_measure_io_stats", "false"},
{"inplace_update_num_locks", "25"}, {"inplace_update_num_locks", "25"},
{"memtable_prefix_bloom_bits", "26"}, {"memtable_prefix_bloom_size_ratio", "0.26"},
{"memtable_prefix_bloom_probes", "27"},
{"memtable_prefix_bloom_huge_page_tlb_size", "28"}, {"memtable_prefix_bloom_huge_page_tlb_size", "28"},
{"bloom_locality", "29"}, {"bloom_locality", "29"},
{"max_successive_merges", "30"}, {"max_successive_merges", "30"},
@ -187,8 +186,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
static_cast<uint64_t>(24)); static_cast<uint64_t>(24));
ASSERT_EQ(new_cf_opt.inplace_update_support, true); ASSERT_EQ(new_cf_opt.inplace_update_support, true);
ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U); ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 26U); ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_probes, 27U);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_huge_page_tlb_size, 28U); ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_huge_page_tlb_size, 28U);
ASSERT_EQ(new_cf_opt.bloom_locality, 29U); ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U); ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
@ -300,10 +298,8 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
const int64_t tera = 1024 * giga; const int64_t tera = 1024 * giga;
// Units (k) // Units (k)
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, ASSERT_OK(GetColumnFamilyOptionsFromString(
"memtable_prefix_bloom_bits=14k;max_write_buffer_number=-15K", base_cf_opt, "max_write_buffer_number=-15K", &new_cf_opt));
&new_cf_opt));
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 14UL * kilo);
ASSERT_EQ(new_cf_opt.max_write_buffer_number, -15 * kilo); ASSERT_EQ(new_cf_opt.max_write_buffer_number, -15 * kilo);
// Units (m) // Units (m)
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,

4
util/testutil.cc
Unescape View File

@ -307,6 +307,8 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
// double options // double options
cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13; cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13; cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->memtable_prefix_bloom_size_ratio =
static_cast<double>(rnd->Uniform(10000)) / 20000.0;
// int options // int options
cf_opt->expanded_compaction_factor = rnd->Uniform(100); cf_opt->expanded_compaction_factor = rnd->Uniform(100);
@ -332,8 +334,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
// uint32_t options // uint32_t options
cf_opt->bloom_locality = rnd->Uniform(10000); cf_opt->bloom_locality = rnd->Uniform(10000);
cf_opt->memtable_prefix_bloom_bits = rnd->Uniform(10000);
cf_opt->memtable_prefix_bloom_probes = rnd->Uniform(10000);
cf_opt->min_partial_merge_operands = rnd->Uniform(10000); cf_opt->min_partial_merge_operands = rnd->Uniform(10000);
cf_opt->max_bytes_for_level_base = rnd->Uniform(10000); cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);

Write Preview
Loading…
Cancel
Save