NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

memtable_prefix_bloom_bits -> memtable_prefix_bloom_bits_ratio and deprecate memtable_prefix_bloom_probes

Browse Source 
Summary:
memtable_prefix_bloom_probes is not a critical option. Remove it to reduce number of options.
It's easier for users to make mistakes with memtable_prefix_bloom_bits, turn it to memtable_prefix_bloom_bits_ratio

Test Plan: Run all existing tests

Reviewers: yhchiang, igor, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: gunnarku, yoshinorim, MarkCallaghan, leveldb, andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D59199
main
sdong 9 years ago
parent e9c1face6c
commit 20699df843
24 changed files with 127 additions and 249 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      HISTORY.md
  2. 11
      db/c.cc
  3. 6
      db/column_family.cc
  4. 3
      db/db_bloom_filter_test.cc
  5. 45
      db/memtable.cc
  6. 1
      db/memtable.h
  7. 7
      db/prefix_test.cc
  8. 1
      db/version_set.h
  9. 13
      include/rocksdb/options.h
  10. 91
      java/rocksjni/options.cc
  11. 30
      java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
  12. 25
      java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java
  13. 30
      java/src/main/java/org/rocksdb/Options.java
  14. 17
      java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
  15. 17
      java/src/test/java/org/rocksdb/OptionsTest.java
  16. 7
      tools/db_bench_tool.cc
  17. 6
      util/mutable_cf_options.cc
  18. 10
      util/mutable_cf_options.h
  19. 13
      util/options.cc
  20. 12
      util/options_helper.cc
  21. 9
      util/options_helper.h
  22. 3
      util/options_settable_test.cc
  23. 12
      util/options_test.cc
  24. 4
      util/testutil.cc

3
HISTORY.md
Unescape View File

@ -1,5 +1,8 @@
# Rocksdb Change Log
## Unreleased
### Public API Change
* Deprecate BlockBaseTableOptions.hash_index_allow_collision=false
* options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes
## 4.9.0 (6/9/2016)
### Public API changes

11
db/c.cc
Unescape View File

@ -1799,14 +1799,9 @@ void rocksdb_options_set_memtable_vector_rep(rocksdb_options_t *opt) {
opt->rep.memtable_factory.reset(factory);
}
void rocksdb_options_set_memtable_prefix_bloom_bits(
rocksdb_options_t* opt, uint32_t v) {
opt->rep.memtable_prefix_bloom_bits = v;
}
void rocksdb_options_set_memtable_prefix_bloom_probes(
rocksdb_options_t* opt, uint32_t v) {
opt->rep.memtable_prefix_bloom_probes = v;
void rocksdb_options_set_memtable_prefix_bloom_size_ratio(
rocksdb_options_t* opt, double v) {
opt->rep.memtable_prefix_bloom_size_ratio = v;
}
void rocksdb_options_set_memtable_prefix_bloom_huge_page_tlb_size(

6
db/column_family.cc
Unescape View File

@ -177,6 +177,12 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
if (result.max_write_buffer_number_to_maintain < 0) {
result.max_write_buffer_number_to_maintain = result.max_write_buffer_number;
}
// bloom filter size shouldn't exceed 1/4 of memtable size.
if (result.memtable_prefix_bloom_size_ratio > 0.25) {
result.memtable_prefix_bloom_size_ratio = 0.25;
} else if (result.memtable_prefix_bloom_size_ratio < 0) {
result.memtable_prefix_bloom_size_ratio = 0;
}
XFUNC_TEST("memtablelist_history", "transaction_xftest_SanitizeOptions",
xf_transaction_set_memtable_history1,
xf_transaction_set_memtable_history,

3
db/db_bloom_filter_test.cc
Unescape View File

@ -615,7 +615,8 @@ class BloomStatsTestWithParam
options_.create_if_missing = true;
options_.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(4));
options_.memtable_prefix_bloom_bits = 8 * 1024;
options_.memtable_prefix_bloom_size_ratio =
8.0 * 1024.0 / static_cast<double>(options_.write_buffer_size);
if (use_block_table_) {
BlockBasedTableOptions table_options;
table_options.hash_index_allow_collision = false;

45
db/memtable.cc
Unescape View File

@ -34,24 +34,25 @@
namespace rocksdb {
MemTableOptions::MemTableOptions(
const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options)
: write_buffer_size(mutable_cf_options.write_buffer_size),
arena_block_size(mutable_cf_options.arena_block_size),
memtable_prefix_bloom_bits(mutable_cf_options.memtable_prefix_bloom_bits),
memtable_prefix_bloom_probes(
mutable_cf_options.memtable_prefix_bloom_probes),
memtable_prefix_bloom_huge_page_tlb_size(
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size),
inplace_update_support(ioptions.inplace_update_support),
inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks),
inplace_callback(ioptions.inplace_callback),
max_successive_merges(mutable_cf_options.max_successive_merges),
filter_deletes(mutable_cf_options.filter_deletes),
statistics(ioptions.statistics),
merge_operator(ioptions.merge_operator),
info_log(ioptions.info_log) {}
MemTableOptions::MemTableOptions(const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options)
: write_buffer_size(mutable_cf_options.write_buffer_size),
arena_block_size(mutable_cf_options.arena_block_size),
memtable_prefix_bloom_bits(
static_cast<uint32_t>(
static_cast<double>(mutable_cf_options.write_buffer_size) *
mutable_cf_options.memtable_prefix_bloom_size_ratio) *
8u),
memtable_prefix_bloom_huge_page_tlb_size(
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size),
inplace_update_support(ioptions.inplace_update_support),
inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks),
inplace_callback(ioptions.inplace_callback),
max_successive_merges(mutable_cf_options.max_successive_merges),
filter_deletes(mutable_cf_options.filter_deletes),
statistics(ioptions.statistics),
merge_operator(ioptions.merge_operator),
info_log(ioptions.info_log) {}
MemTable::MemTable(const InternalKeyComparator& cmp,
const ImmutableCFOptions& ioptions,
@ -88,11 +89,9 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) {
prefix_bloom_.reset(new DynamicBloom(
&allocator_,
moptions_.memtable_prefix_bloom_bits, ioptions.bloom_locality,
moptions_.memtable_prefix_bloom_probes, nullptr,
moptions_.memtable_prefix_bloom_huge_page_tlb_size,
ioptions.info_log));
&allocator_, moptions_.memtable_prefix_bloom_bits,
ioptions.bloom_locality, 6 /* hard coded 6 probes */, nullptr,
moptions_.memtable_prefix_bloom_huge_page_tlb_size, ioptions.info_log));
}
}

1
db/memtable.h
Unescape View File

@ -42,7 +42,6 @@ struct MemTableOptions {
size_t write_buffer_size;
size_t arena_block_size;
uint32_t memtable_prefix_bloom_bits;
uint32_t memtable_prefix_bloom_probes;
size_t memtable_prefix_bloom_huge_page_tlb_size;
bool inplace_update_support;
size_t inplace_update_num_locks;

7
db/prefix_test.cc
Unescape View File

@ -43,8 +43,7 @@ DEFINE_int64(write_buffer_size, 33554432, "");
DEFINE_int32(max_write_buffer_number, 2, "");
DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
DEFINE_int32(skiplist_height, 4, "");
DEFINE_int32(memtable_prefix_bloom_bits, 10000000, "");
DEFINE_int32(memtable_prefix_bloom_probes, 10, "");
DEFINE_double(memtable_prefix_bloom_size_ratio, 0.1, "");
DEFINE_int32(memtable_prefix_bloom_huge_page_tlb_size, 2 * 1024 * 1024, "");
DEFINE_int32(value_size, 40, "");
@ -160,8 +159,8 @@ class PrefixTest : public testing::Test {
options.min_write_buffer_number_to_merge =
FLAGS_min_write_buffer_number_to_merge;
options.memtable_prefix_bloom_bits = FLAGS_memtable_prefix_bloom_bits;
options.memtable_prefix_bloom_probes = FLAGS_memtable_prefix_bloom_probes;
options.memtable_prefix_bloom_size_ratio =
FLAGS_memtable_prefix_bloom_size_ratio;
options.memtable_prefix_bloom_huge_page_tlb_size =
FLAGS_memtable_prefix_bloom_huge_page_tlb_size;

1
db/version_set.h
Unescape View File

@ -57,7 +57,6 @@ class Version;
class VersionSet;
class WriteBuffer;
class MergeContext;
class ColumnFamilyData;
class ColumnFamilySet;
class TableCache;
class MergeIteratorBuilder;

13
include/rocksdb/options.h
Unescape View File

@ -745,16 +745,15 @@ struct ColumnFamilyOptions {
Slice delta_value,
std::string* merged_value);
// if prefix_extractor is set and bloom_bits is not 0, create prefix bloom
// for memtable
// if prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
// create prefix bloom for memtable with the size of
// write_buffer_size * memtable_prefix_bloom_size_ratio.
// If it is larger than 0.25, it is santinized to 0.25.
//
// Dynamically changeable through SetOptions() API
uint32_t memtable_prefix_bloom_bits;
// number of hash probes per key
// Default: 0 (disable)
//
// Dynamically changeable through SetOptions() API
uint32_t memtable_prefix_bloom_probes;
double memtable_prefix_bloom_size_ratio;
// Page size for huge page TLB for bloom in memtable. If <=0, not allocate
// from huge page TLB but from malloc.

91
java/rocksjni/options.cc
Unescape View File

@ -1732,50 +1732,27 @@ void Java_org_rocksdb_Options_setInplaceUpdateNumLocks(
/*
* Class: org_rocksdb_Options
* Method: memtablePrefixBloomBits
* Method: memtablePrefixBloomSizeRatio
* Signature: (J)I
*/
jint Java_org_rocksdb_Options_memtablePrefixBloomBits(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_bits;
}
/*
* Class: org_rocksdb_Options
* Method: setMemtablePrefixBloomBits
* Signature: (JI)V
*/
void Java_org_rocksdb_Options_setMemtablePrefixBloomBits(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_bits) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_bits =
static_cast<int32_t>(jmemtable_prefix_bloom_bits);
}
/*
* Class: org_rocksdb_Options
* Method: memtablePrefixBloomProbes
* Signature: (J)I
*/
jint Java_org_rocksdb_Options_memtablePrefixBloomProbes(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_probes;
jdouble Java_org_rocksdb_Options_memtablePrefixBloomSizeRatio(JNIEnv* env,
jobject jobj,
jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(jhandle)
->memtable_prefix_bloom_size_ratio;
}
/*
* Class: org_rocksdb_Options
* Method: setMemtablePrefixBloomProbes
* Method: setMemtablePrefixBloomSizeRatio
* Signature: (JI)V
*/
void Java_org_rocksdb_Options_setMemtablePrefixBloomProbes(
void Java_org_rocksdb_Options_setMemtablePrefixBloomSizeRatio(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_probes) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->memtable_prefix_bloom_probes =
static_cast<int32_t>(jmemtable_prefix_bloom_probes);
jdouble jmemtable_prefix_bloom_size_ratio) {
reinterpret_cast<rocksdb::Options*>(jhandle)
->memtable_prefix_bloom_size_ratio =
static_cast<double>(jmemtable_prefix_bloom_size_ratio);
}
/*
@ -2915,50 +2892,26 @@ void Java_org_rocksdb_ColumnFamilyOptions_setInplaceUpdateNumLocks(
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: memtablePrefixBloomBits
* Method: memtablePrefixBloomSizeRatio
* Signature: (J)I
*/
jint Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomBits(
jdouble Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomSizeRatio(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_bits;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMemtablePrefixBloomBits
* Signature: (JI)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomBits(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_bits) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_bits =
static_cast<int32_t>(jmemtable_prefix_bloom_bits);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: memtablePrefixBloomProbes
* Signature: (J)I
*/
jint Java_org_rocksdb_ColumnFamilyOptions_memtablePrefixBloomProbes(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_probes;
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)
->memtable_prefix_bloom_size_ratio;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMemtablePrefixBloomProbes
* Method: setMemtablePrefixBloomSizeRatio
* Signature: (JI)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomProbes(
void Java_org_rocksdb_ColumnFamilyOptions_setMemtablePrefixBloomSizeRatio(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmemtable_prefix_bloom_probes) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->memtable_prefix_bloom_probes =
static_cast<int32_t>(jmemtable_prefix_bloom_probes);
jdouble jmemtable_prefix_bloom_size_ratio) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)
->memtable_prefix_bloom_size_ratio =
static_cast<double>(jmemtable_prefix_bloom_size_ratio);
}
/*

30
java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
Unescape View File

@ -589,27 +589,15 @@ public class ColumnFamilyOptions extends RocksObject
}
@Override
public ColumnFamilyOptions setMemtablePrefixBloomBits(
final int memtablePrefixBloomBits) {
setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomBits);
public ColumnFamilyOptions setMemtablePrefixBloomSizeRatio(
final double memtablePrefixBloomSizeRatio) {
setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomSizeRatio);
return this;
}
@Override
public int memtablePrefixBloomBits() {
return memtablePrefixBloomBits(nativeHandle_);
}
@Override
public ColumnFamilyOptions setMemtablePrefixBloomProbes(
final int memtablePrefixBloomProbes) {
setMemtablePrefixBloomProbes(nativeHandle_, memtablePrefixBloomProbes);
return this;
}
@Override
public int memtablePrefixBloomProbes() {
return memtablePrefixBloomProbes(nativeHandle_);
public double memtablePrefixBloomSizeRatio() {
return memtablePrefixBloomSizeRatio(nativeHandle_);
}
@Override
@ -787,12 +775,8 @@ public class ColumnFamilyOptions extends RocksObject
long handle, long inplaceUpdateNumLocks)
throws IllegalArgumentException;
private native long inplaceUpdateNumLocks(long handle);
private native void setMemtablePrefixBloomBits(
long handle, int memtablePrefixBloomBits);
private native int memtablePrefixBloomBits(long handle);
private native void setMemtablePrefixBloomProbes(
long handle, int memtablePrefixBloomProbes);
private native int memtablePrefixBloomProbes(long handle);
private native void setMemtablePrefixBloomBits(long handle, double memtablePrefixBloomSizeRatio);
private native double memtablePrefixBloomSizeRatio(long handle);
private native void setBloomLocality(
long handle, int bloomLocality);
private native int bloomLocality(long handle);

25
java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java
Unescape View File

@ -1011,15 +1011,15 @@ public interface ColumnFamilyOptionsInterface {
long inplaceUpdateNumLocks();
/**
* Sets the number of bits used in the prefix bloom filter.
* Sets the size ratio of the memtable used in the prefix bloom filter.
*
* This value will be used only when a prefix-extractor is specified.
*
* @param memtablePrefixBloomBits the number of bits used in the
* @param memtablePrefixBloomSizeRatio the number of bits used in the
* prefix bloom filter.
* @return the reference to the current option.
*/
Object setMemtablePrefixBloomBits(int memtablePrefixBloomBits);
Object setMemtablePrefixBloomSizeRatio(double memtablePrefixBloomSizeRatio);
/**
* Returns the number of bits used in the prefix bloom filter.
@ -1029,22 +1029,7 @@ public interface ColumnFamilyOptionsInterface {
* @return the number of bloom-bits.
* @see #useFixedLengthPrefixExtractor(int)
*/
int memtablePrefixBloomBits();
/**
* The number of hash probes per key used in the mem-table.
*
* @param memtablePrefixBloomProbes the number of hash probes per key.
* @return the reference to the current option.
*/
Object setMemtablePrefixBloomProbes(int memtablePrefixBloomProbes);
/**
* The number of hash probes per key used in the mem-table.
*
* @return the number of hash probes per key.
*/
int memtablePrefixBloomProbes();
double memtablePrefixBloomSizeRatio();
/**
* Control locality of bloom filter probes to improve cache miss rate.
@ -1074,7 +1059,7 @@ public interface ColumnFamilyOptionsInterface {
* Default: 0
*
* @return the level of locality of bloom-filter probes.
* @see #setMemtablePrefixBloomProbes(int)
* @see #setBloomLocality(int)
*/
int bloomLocality();

30
java/src/main/java/org/rocksdb/Options.java
Unescape View File

@ -1005,26 +1005,13 @@ public class Options extends RocksObject
}
@Override
public int memtablePrefixBloomBits() {
return memtablePrefixBloomBits(nativeHandle_);
public double memtablePrefixBloomSizeRatio() {
return memtablePrefixBloomSizeRatio(nativeHandle_);
}
@Override
public Options setMemtablePrefixBloomBits(
final int memtablePrefixBloomBits) {
setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomBits);
return this;
}
@Override
public int memtablePrefixBloomProbes() {
return memtablePrefixBloomProbes(nativeHandle_);
}
@Override
public Options setMemtablePrefixBloomProbes(
final int memtablePrefixBloomProbes) {
setMemtablePrefixBloomProbes(nativeHandle_, memtablePrefixBloomProbes);
public Options setMemtablePrefixBloomSizeRatio(final double memtablePrefixBloomSizeRatio) {
setMemtablePrefixBloomSizeRatio(nativeHandle_, memtablePrefixBloomSizeRatio);
return this;
}
@ -1293,12 +1280,9 @@ public class Options extends RocksObject
long handle, long inplaceUpdateNumLocks)
throws IllegalArgumentException;
private native long inplaceUpdateNumLocks(long handle);
private native void setMemtablePrefixBloomBits(
long handle, int memtablePrefixBloomBits);
private native int memtablePrefixBloomBits(long handle);
private native void setMemtablePrefixBloomProbes(
long handle, int memtablePrefixBloomProbes);
private native int memtablePrefixBloomProbes(long handle);
private native void setMemtablePrefixBloomSizeRatio(
long handle, double memtablePrefixBloomSizeRatio);
private native double memtablePrefixBloomSizeRatio(long handle);
private native void setBloomLocality(
long handle, int bloomLocality);
private native int bloomLocality(long handle);

17
java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
Unescape View File

@ -305,20 +305,11 @@ public class ColumnFamilyOptionsTest {
}
@Test
public void memtablePrefixBloomBits() {
public void memtablePrefixBloomSizeRatio() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomBits(intValue);
assertThat(opt.memtablePrefixBloomBits()).isEqualTo(intValue);
}
}
@Test
public void memtablePrefixBloomProbes() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomProbes(intValue);
assertThat(opt.memtablePrefixBloomProbes()).isEqualTo(intValue);
final double doubleValue = rand.nextDouble();
opt.setMemtablePrefixBloomSizeRatio(doubleValue);
assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue);
}
}

17
java/src/test/java/org/rocksdb/OptionsTest.java
Unescape View File

@ -268,20 +268,11 @@ public class OptionsTest {
}
@Test
public void memtablePrefixBloomBits() {
public void memtablePrefixBloomSizeRatio() {
try (final Options opt = new Options()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomBits(intValue);
assertThat(opt.memtablePrefixBloomBits()).isEqualTo(intValue);
}
}
@Test
public void memtablePrefixBloomProbes() {
try (final Options opt = new Options()) {
final int intValue = rand.nextInt();
opt.setMemtablePrefixBloomProbes(intValue);
assertThat(opt.memtablePrefixBloomProbes()).isEqualTo(intValue);
final double doubleValue = rand.nextDouble();
opt.setMemtablePrefixBloomSizeRatio(doubleValue);
assertThat(opt.memtablePrefixBloomSizeRatio()).isEqualTo(doubleValue);
}
}

7
tools/db_bench_tool.cc
Unescape View File

@ -396,8 +396,9 @@ DEFINE_int32(skip_table_builder_flush, false, "Skip flushing block in "
DEFINE_int32(bloom_bits, -1, "Bloom filter bits per key. Negative means"
" use default settings.");
DEFINE_int32(memtable_bloom_bits, 0, "Bloom filter bits per key for memtable. "
"Negative means no bloom filter.");
DEFINE_double(memtable_bloom_size_ratio, 0,
"Ratio of memtable size used for bloom filter. 0 means no bloom "
"filter.");
DEFINE_bool(use_existing_db, false, "If true, do not destroy the existing"
" database. If you set this flag and also specify a benchmark that"
@ -2461,7 +2462,7 @@ class Benchmark {
exit(1);
}
}
options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits;
options.memtable_prefix_bloom_size_ratio = FLAGS_memtable_bloom_size_ratio;
options.bloom_locality = FLAGS_bloom_locality;
options.max_file_opening_threads = FLAGS_file_opening_threads;
options.new_table_reader_for_compaction_inputs =

6
util/mutable_cf_options.cc
Unescape View File

@ -70,10 +70,8 @@ void MutableCFOptions::Dump(Logger* log) const {
max_write_buffer_number);
Log(log, " arena_block_size: %" ROCKSDB_PRIszt,
arena_block_size);
Log(log, " memtable_prefix_bloom_bits: %" PRIu32,
memtable_prefix_bloom_bits);
Log(log, " memtable_prefix_bloom_probes: %" PRIu32,
memtable_prefix_bloom_probes);
Log(log, " memtable_prefix_bloom_ratio: %f",
memtable_prefix_bloom_size_ratio);
Log(log, " memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt,
memtable_prefix_bloom_huge_page_tlb_size);
Log(log, " max_successive_merges: %" ROCKSDB_PRIszt,

10
util/mutable_cf_options.h
Unescape View File

@ -17,8 +17,8 @@ struct MutableCFOptions {
: write_buffer_size(options.write_buffer_size),
max_write_buffer_number(options.max_write_buffer_number),
arena_block_size(options.arena_block_size),
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits),
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes),
memtable_prefix_bloom_size_ratio(
options.memtable_prefix_bloom_size_ratio),
memtable_prefix_bloom_huge_page_tlb_size(
options.memtable_prefix_bloom_huge_page_tlb_size),
max_successive_merges(options.max_successive_merges),
@ -58,8 +58,7 @@ struct MutableCFOptions {
: write_buffer_size(0),
max_write_buffer_number(0),
arena_block_size(0),
memtable_prefix_bloom_bits(0),
memtable_prefix_bloom_probes(0),
memtable_prefix_bloom_size_ratio(0),
memtable_prefix_bloom_huge_page_tlb_size(0),
max_successive_merges(0),
filter_deletes(false),
@ -110,8 +109,7 @@ struct MutableCFOptions {
size_t write_buffer_size;
int max_write_buffer_number;
size_t arena_block_size;
uint32_t memtable_prefix_bloom_bits;
uint32_t memtable_prefix_bloom_probes;
double memtable_prefix_bloom_size_ratio;
size_t memtable_prefix_bloom_huge_page_tlb_size;
size_t max_successive_merges;
bool filter_deletes;

13
util/options.cc
Unescape View File

@ -121,8 +121,7 @@ ColumnFamilyOptions::ColumnFamilyOptions()
inplace_update_support(false),
inplace_update_num_locks(10000),
inplace_callback(nullptr),
memtable_prefix_bloom_bits(0),
memtable_prefix_bloom_probes(6),
memtable_prefix_bloom_size_ratio(0.0),
memtable_prefix_bloom_huge_page_tlb_size(0),
bloom_locality(0),
max_successive_merges(0),
@ -190,8 +189,8 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
inplace_update_support(options.inplace_update_support),
inplace_update_num_locks(options.inplace_update_num_locks),
inplace_callback(options.inplace_callback),
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits),
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes),
memtable_prefix_bloom_size_ratio(
options.memtable_prefix_bloom_size_ratio),
memtable_prefix_bloom_huge_page_tlb_size(
options.memtable_prefix_bloom_huge_page_tlb_size),
bloom_locality(options.bloom_locality),
@ -596,10 +595,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
Header(log, " Options.min_partial_merge_operands: %u",
min_partial_merge_operands);
// TODO: easier config for bloom (maybe based on avg key/value size)
Header(log, " Options.memtable_prefix_bloom_bits: %d",
memtable_prefix_bloom_bits);
Header(log, " Options.memtable_prefix_bloom_probes: %d",
memtable_prefix_bloom_probes);
Header(log, " Options.memtable_prefix_bloom_size_ratio: %f",
memtable_prefix_bloom_size_ratio);
Header(log,
" Options.memtable_prefix_bloom_huge_page_tlb_size: %" ROCKSDB_PRIszt,

12
util/options_helper.cc
Unescape View File

@ -537,9 +537,11 @@ bool ParseMemtableOptions(const std::string& name, const std::string& value,
} else if (name == "arena_block_size") {
new_options->arena_block_size = ParseSizeT(value);
} else if (name == "memtable_prefix_bloom_bits") {
new_options->memtable_prefix_bloom_bits = ParseUint32(value);
// deprecated
} else if (name == "memtable_prefix_bloom_size_ratio") {
new_options->memtable_prefix_bloom_size_ratio = ParseDouble(value);
} else if (name == "memtable_prefix_bloom_probes") {
new_options->memtable_prefix_bloom_probes = ParseUint32(value);
// Deprecated
} else if (name == "memtable_prefix_bloom_huge_page_tlb_size") {
new_options->memtable_prefix_bloom_huge_page_tlb_size =
ParseSizeT(value);
@ -1437,10 +1439,8 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size;
cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number;
cf_opts.arena_block_size = mutable_cf_options.arena_block_size;
cf_opts.memtable_prefix_bloom_bits =
mutable_cf_options.memtable_prefix_bloom_bits;
cf_opts.memtable_prefix_bloom_probes =
mutable_cf_options.memtable_prefix_bloom_probes;
cf_opts.memtable_prefix_bloom_size_ratio =
mutable_cf_options.memtable_prefix_bloom_size_ratio;
cf_opts.memtable_prefix_bloom_huge_page_tlb_size =
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size;
cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges;

9
util/options_helper.h
Unescape View File

@ -437,11 +437,12 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{offsetof(struct ColumnFamilyOptions, bloom_locality),
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{"memtable_prefix_bloom_bits",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_bits),
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated}},
{"memtable_prefix_bloom_size_ratio",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_size_ratio),
OptionType::kDouble, OptionVerificationType::kNormal}},
{"memtable_prefix_bloom_probes",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_probes),
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated}},
{"min_partial_merge_operands",
{offsetof(struct ColumnFamilyOptions, min_partial_merge_operands),
OptionType::kUInt32T, OptionVerificationType::kNormal}},

3
util/options_settable_test.cc
Unescape View File

@ -413,14 +413,13 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"max_write_buffer_number_to_maintain=84;"
"verify_checksums_in_compaction=false;"
"merge_operator=aabcxehazrMergeOperator;"
"memtable_prefix_bloom_bits=4642;"
"memtable_prefix_bloom_size_ratio=0.4642;"
"paranoid_file_checks=true;"
"inplace_update_num_locks=7429;"
"optimize_filters_for_hits=false;"
"level_compaction_dynamic_level_bytes=false;"
"inplace_update_support=false;"
"compaction_style=kCompactionStyleFIFO;"
"memtable_prefix_bloom_probes=2511;"
"purge_redundant_kvs_while_flush=true;"
"filter_deletes=false;"
"hard_pending_compaction_bytes_limit=0;"

12
util/options_test.cc
Unescape View File

@ -86,8 +86,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"report_bg_io_stats", "true"},
{"compaction_measure_io_stats", "false"},
{"inplace_update_num_locks", "25"},
{"memtable_prefix_bloom_bits", "26"},
{"memtable_prefix_bloom_probes", "27"},
{"memtable_prefix_bloom_size_ratio", "0.26"},
{"memtable_prefix_bloom_huge_page_tlb_size", "28"},
{"bloom_locality", "29"},
{"max_successive_merges", "30"},
@ -187,8 +186,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
static_cast<uint64_t>(24));
ASSERT_EQ(new_cf_opt.inplace_update_support, true);
ASSERT_EQ(new_cf_opt.inplace_update_num_locks, 25U);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 26U);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_probes, 27U);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_size_ratio, 0.26);
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_huge_page_tlb_size, 28U);
ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
@ -300,10 +298,8 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
const int64_t tera = 1024 * giga;
// Units (k)
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
"memtable_prefix_bloom_bits=14k;max_write_buffer_number=-15K",
&new_cf_opt));
ASSERT_EQ(new_cf_opt.memtable_prefix_bloom_bits, 14UL * kilo);
ASSERT_OK(GetColumnFamilyOptionsFromString(
base_cf_opt, "max_write_buffer_number=-15K", &new_cf_opt));
ASSERT_EQ(new_cf_opt.max_write_buffer_number, -15 * kilo);
// Units (m)
ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,

4
util/testutil.cc
Unescape View File

@ -307,6 +307,8 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
// double options
cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
cf_opt->memtable_prefix_bloom_size_ratio =
static_cast<double>(rnd->Uniform(10000)) / 20000.0;
// int options
cf_opt->expanded_compaction_factor = rnd->Uniform(100);
@ -332,8 +334,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
// uint32_t options
cf_opt->bloom_locality = rnd->Uniform(10000);
cf_opt->memtable_prefix_bloom_bits = rnd->Uniform(10000);
cf_opt->memtable_prefix_bloom_probes = rnd->Uniform(10000);
cf_opt->min_partial_merge_operands = rnd->Uniform(10000);
cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);

Write Preview
Loading…
Cancel
Save