NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[rocksdb][PR] Remove option min_partial_merge_operands and verify_checksums_in_comp…

Browse Source 
Summary:
…action

 The two options, min_partial_merge_operands and verify_checksums_in_compaction, are not seldom used. Remove them to reduce the total number of options. Also remove them from Java and C interface.
Closes https://github.com/facebook/rocksdb/pull/1902

Differential Revision: D4601219

Pulled By: siying

fbshipit-source-id: aad4cb2
main
Siying Dong 8 years ago committed by Facebook Github Bot
parent 1ba2804b7f
commit 8efb5ffa2a
31 changed files with 30 additions and 367 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      HISTORY.md
  2. 1
      db/builder.cc
  3. 10
      db/c.cc
  4. 2
      db/compaction_iterator_test.cc
  5. 1
      db/compaction_job.cc
  6. 6
      db/db_test.cc
  7. 3
      db/merge_helper.cc
  8. 3
      db/merge_helper.h
  9. 2
      db/merge_helper_test.cc
  10. 32
      db/merge_test.cc
  11. 3
      db/version_set.cc
  12. 5
      include/rocksdb/c.h
  13. 6
      include/rocksdb/merge_operator.h
  14. 16
      include/rocksdb/options.h
  15. 96
      java/rocksjni/options.cc
  16. 31
      java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
  17. 27
      java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java
  18. 15
      java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
  19. 20
      java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java
  20. 31
      java/src/main/java/org/rocksdb/Options.java
  21. 18
      java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
  22. 10
      java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java
  23. 18
      java/src/test/java/org/rocksdb/OptionsTest.java
  24. 4
      util/cf_options.cc
  25. 10
      util/cf_options.h
  26. 6
      util/options.cc
  27. 5
      util/options_helper.cc
  28. 8
      util/options_helper.h
  29. 2
      util/options_settable_test.cc
  30. 2
      util/options_test.cc
  31. 2
      util/testutil.cc

2
HISTORY.md
Unescape View File

@ -3,6 +3,8 @@
### Public API Change
* Remove disableDataSync option.
* Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0.
* Remove option min_partial_merge_operands. Partial merge operands will always be merged in flush or compaction if there are more than one.
* Remove option verify_checksums_in_compaction. Compaction will always verify checksum.
## 5.2.0 (02/08/2017)
### Public API Change

1
db/builder.cc
Unescape View File

@ -122,7 +122,6 @@ Status BuildTable(
MergeHelper merge(env, internal_comparator.user_comparator(),
ioptions.merge_operator, nullptr, ioptions.info_log,
mutable_cf_options.min_partial_merge_operands,
true /* internal key corruption is not ok */,
snapshots.empty() ? 0 : snapshots.back());

10
db/c.cc
Unescape View File

@ -1824,11 +1824,6 @@ void rocksdb_options_set_enable_write_thread_adaptive_yield(
opt->rep.enable_write_thread_adaptive_yield = v;
}
void rocksdb_options_set_verify_checksums_in_compaction(
rocksdb_options_t* opt, unsigned char v) {
opt->rep.verify_checksums_in_compaction = v;
}
void rocksdb_options_set_max_sequential_skip_in_iterations(
rocksdb_options_t* opt, uint64_t v) {
opt->rep.max_sequential_skip_in_iterations = v;
@ -1980,11 +1975,6 @@ void rocksdb_options_set_max_successive_merges(
opt->rep.max_successive_merges = v;
}
void rocksdb_options_set_min_partial_merge_operands(
rocksdb_options_t* opt, uint32_t v) {
opt->rep.min_partial_merge_operands = v;
}
void rocksdb_options_set_bloom_locality(
rocksdb_options_t* opt, uint32_t v) {
opt->rep.bloom_locality = v;

2
db/compaction_iterator_test.cc
Unescape View File

@ -182,7 +182,7 @@ class CompactionIteratorTest : public testing::Test {
}
merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, merge_op, filter,
nullptr, 0U, false, 0, 0, nullptr,
nullptr, false, 0, 0, nullptr,
&shutting_down_));
iter_.reset(new LoggingForwardVectorIterator(ks, vs));
iter_->SeekToFirst();

1
db/compaction_job.cc
Unescape View File

@ -721,7 +721,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
MergeHelper merge(
env_, cfd->user_comparator(), cfd->ioptions()->merge_operator,
compaction_filter, db_options_.info_log.get(),
mutable_cf_options->min_partial_merge_operands,
false /* internal key corruption is expected */,
existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
compact_->compaction->level(), db_options_.statistics.get(),

6
db/db_test.cc
Unescape View File

@ -4691,12 +4691,6 @@ TEST_F(DBTest, DynamicMiscOptions) {
ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1],
&mutable_cf_options));
ASSERT_EQ(true, mutable_cf_options.report_bg_io_stats);
// Test min_partial_merge_operands
ASSERT_OK(
dbfull()->SetOptions(handles_[1], {{"min_partial_merge_operands", "4"}}));
ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1],
&mutable_cf_options));
ASSERT_EQ(4, mutable_cf_options.min_partial_merge_operands);
// Test compression
// sanity check
ASSERT_OK(dbfull()->SetOptions({{"compression", "kNoCompression"}}));

3
db/merge_helper.cc
Unescape View File

@ -283,8 +283,7 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
// Attempt to use the user's associative merge function to
// merge the stacked merge operands into a single operand.
s = Status::MergeInProgress();
if (merge_context_.GetNumOperands() >= 2 &&
merge_context_.GetNumOperands() >= min_partial_merge_operands_) {
if (merge_context_.GetNumOperands() >= 2) {
bool merge_success = false;
std::string merge_result;
{

3
db/merge_helper.h
Unescape View File

@ -32,7 +32,6 @@ class MergeHelper {
MergeHelper(Env* env, const Comparator* user_comparator,
const MergeOperator* user_merge_operator,
const CompactionFilter* compaction_filter, Logger* logger,
unsigned min_partial_merge_operands,
bool assert_valid_internal_key, SequenceNumber latest_snapshot,
int level = 0, Statistics* stats = nullptr,
const std::atomic<bool>* shutting_down = nullptr)
@ -42,7 +41,6 @@ class MergeHelper {
compaction_filter_(compaction_filter),
shutting_down_(shutting_down),
logger_(logger),
min_partial_merge_operands_(min_partial_merge_operands),
assert_valid_internal_key_(assert_valid_internal_key),
latest_snapshot_(latest_snapshot),
level_(level),
@ -156,7 +154,6 @@ class MergeHelper {
const CompactionFilter* compaction_filter_;
const std::atomic<bool>* shutting_down_;
Logger* logger_;
unsigned min_partial_merge_operands_;
bool assert_valid_internal_key_; // enforce no internal key corruption?
SequenceNumber latest_snapshot_;
int level_;

2
db/merge_helper_test.cc
Unescape View File

@ -28,7 +28,7 @@ class MergeHelperTest : public testing::Test {
iter_->SeekToFirst();
merge_helper_.reset(new MergeHelper(env_, BytewiseComparator(),
merge_op_.get(), filter_.get(), nullptr,
2U, false, latest_snapshot));
false, latest_snapshot));
return merge_helper_->MergeUntil(iter_.get(), nullptr /* range_del_agg */,
stop_before, at_bottom);
}

32
db/merge_test.cc
Unescape View File

@ -76,14 +76,12 @@ class CountMergeOperator : public AssociativeMergeOperator {
namespace {
std::shared_ptr<DB> OpenDb(const std::string& dbname, const bool ttl = false,
const size_t max_successive_merges = 0,
const uint32_t min_partial_merge_operands = 2) {
const size_t max_successive_merges = 0) {
DB* db;
Options options;
options.create_if_missing = true;
options.merge_operator = std::make_shared<CountMergeOperator>();
options.max_successive_merges = max_successive_merges;
options.min_partial_merge_operands = min_partial_merge_operands;
Status s;
DestroyDB(dbname, Options());
// DBWithTTL is not supported in ROCKSDB_LITE
@ -448,20 +446,20 @@ void runTest(int argc, const std::string& dbname, const bool use_ttl = false) {
{
std::cout << "Test Partial-Merge\n";
size_t max_merge = 100;
for (uint32_t min_merge = 5; min_merge < 25; min_merge += 5) {
for (uint32_t count = min_merge - 1; count <= min_merge + 1; count++) {
auto db = OpenDb(dbname, use_ttl, max_merge, min_merge);
MergeBasedCounters counters(db, 0);
testPartialMerge(&counters, db.get(), max_merge, min_merge, count);
DestroyDB(dbname, Options());
}
{
auto db = OpenDb(dbname, use_ttl, max_merge, min_merge);
MergeBasedCounters counters(db, 0);
testPartialMerge(&counters, db.get(), max_merge, min_merge,
min_merge * 10);
DestroyDB(dbname, Options());
}
// Min merge is hard-coded to 2.
uint32_t min_merge = 2;
for (uint32_t count = min_merge - 1; count <= min_merge + 1; count++) {
auto db = OpenDb(dbname, use_ttl, max_merge);
MergeBasedCounters counters(db, 0);
testPartialMerge(&counters, db.get(), max_merge, min_merge, count);
DestroyDB(dbname, Options());
}
{
auto db = OpenDb(dbname, use_ttl, max_merge);
MergeBasedCounters counters(db, 0);
testPartialMerge(&counters, db.get(), max_merge, min_merge,
min_merge * 10);
DestroyDB(dbname, Options());
}
}

3
db/version_set.cc
Unescape View File

@ -3431,8 +3431,7 @@ InternalIterator* VersionSet::MakeInputIterator(
const Compaction* c, RangeDelAggregator* range_del_agg) {
auto cfd = c->column_family_data();
ReadOptions read_options;
read_options.verify_checksums =
c->mutable_cf_options()->verify_checksums_in_compaction;
read_options.verify_checksums = true;
read_options.fill_cache = false;
if (c->ShouldFormSubcompactions()) {
read_options.total_order_seek = true;

5
include/rocksdb/c.h
Unescape View File

@ -700,9 +700,6 @@ extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_enable_write_thread_adaptive_yield(rocksdb_options_t*,
unsigned char);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_verify_checksums_in_compaction(rocksdb_options_t*,
unsigned char);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_max_sequential_skip_in_iterations(rocksdb_options_t*,
uint64_t);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_disable_auto_compactions(
@ -735,8 +732,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_huge_page_size(
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_successive_merges(
rocksdb_options_t*, size_t);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_partial_merge_operands(
rocksdb_options_t*, uint32_t);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bloom_locality(
rocksdb_options_t*, uint32_t);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_support(

6
include/rocksdb/merge_operator.h
Unescape View File

@ -165,10 +165,8 @@ class MergeOperator {
//
// The string that new_value is pointing to will be empty.
//
// The PartialMergeMulti function will be called only when the list of
// operands are long enough. The minimum amount of operands that will be
// passed to the function are specified by the "min_partial_merge_operands"
// option.
// The PartialMergeMulti function will be called when there are at least two
// operands.
//
// In the default implementation, PartialMergeMulti will invoke PartialMerge
// multiple times, where each time it only merges two operands. Developers

16
include/rocksdb/options.h
Unescape View File

@ -615,13 +615,6 @@ struct ColumnFamilyOptions {
// Default: kByCompensatedSize
CompactionPri compaction_pri = kByCompensatedSize;
// If true, compaction will verify checksum on every read that happens
// as part of compaction
//
// Default: true
//
// Dynamically changeable through SetOptions() API
bool verify_checksums_in_compaction = true;
// The options needed to support Universal Style compactions
CompactionOptionsUniversal compaction_options_universal;
@ -798,15 +791,6 @@ struct ColumnFamilyOptions {
// Dynamically changeable through SetOptions() API
size_t max_successive_merges = 0;
// The number of partial merge operands to accumulate before partial
// merge will be performed. Partial merge will not be called
// if the list of values to merge is less than min_partial_merge_operands.
//
// If min_partial_merge_operands < 2, then it will be treated as 2.
//
// Default: 2
uint32_t min_partial_merge_operands = 2;
// This flag specifies that the implementation should optimize the filters
// mainly for cases where keys are found rather than also optimize for keys
// missed. This would be used in cases where the application knows that

96
java/rocksjni/options.cc
Unescape View File

@ -1707,30 +1707,6 @@ void Java_org_rocksdb_Options_setPurgeRedundantKvsWhileFlush(
static_cast<bool>(jpurge_redundant_kvs_while_flush);
}
/*
* Class: org_rocksdb_Options
* Method: verifyChecksumsInCompaction
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_Options_verifyChecksumsInCompaction(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->verify_checksums_in_compaction;
}
/*
* Class: org_rocksdb_Options
* Method: setVerifyChecksumsInCompaction
* Signature: (JZ)V
*/
void Java_org_rocksdb_Options_setVerifyChecksumsInCompaction(
JNIEnv* env, jobject jobj, jlong jhandle,
jboolean jverify_checksums_in_compaction) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->verify_checksums_in_compaction =
static_cast<bool>(jverify_checksums_in_compaction);
}
/*
* Class: org_rocksdb_Options
* Method: maxSequentialSkipInIterations
@ -1882,30 +1858,6 @@ void Java_org_rocksdb_Options_setMaxSuccessiveMerges(
}
}
/*
* Class: org_rocksdb_Options
* Method: minPartialMergeOperands
* Signature: (J)I
*/
jint Java_org_rocksdb_Options_minPartialMergeOperands(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->min_partial_merge_operands;
}
/*
* Class: org_rocksdb_Options
* Method: setMinPartialMergeOperands
* Signature: (JI)V
*/
void Java_org_rocksdb_Options_setMinPartialMergeOperands(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmin_partial_merge_operands) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->min_partial_merge_operands =
static_cast<int32_t>(jmin_partial_merge_operands);
}
/*
* Class: org_rocksdb_Options
* Method: optimizeFiltersForHits
@ -3013,30 +2965,6 @@ void Java_org_rocksdb_ColumnFamilyOptions_setPurgeRedundantKvsWhileFlush(
static_cast<bool>(jpurge_redundant_kvs_while_flush);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: verifyChecksumsInCompaction
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_ColumnFamilyOptions_verifyChecksumsInCompaction(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->verify_checksums_in_compaction;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setVerifyChecksumsInCompaction
* Signature: (JZ)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setVerifyChecksumsInCompaction(
JNIEnv* env, jobject jobj, jlong jhandle,
jboolean jverify_checksums_in_compaction) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->verify_checksums_in_compaction =
static_cast<bool>(jverify_checksums_in_compaction);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: maxSequentialSkipInIterations
@ -3189,30 +3117,6 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxSuccessiveMerges(
}
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: minPartialMergeOperands
* Signature: (J)I
*/
jint Java_org_rocksdb_ColumnFamilyOptions_minPartialMergeOperands(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->min_partial_merge_operands;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMinPartialMergeOperands
* Signature: (JI)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMinPartialMergeOperands(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmin_partial_merge_operands) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->min_partial_merge_operands =
static_cast<int32_t>(jmin_partial_merge_operands);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: optimizeFiltersForHits

31
java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
Unescape View File

@ -473,19 +473,6 @@ public class ColumnFamilyOptions extends RocksObject
return maxTableFilesSizeFIFO(nativeHandle_);
}
@Override
public ColumnFamilyOptions setVerifyChecksumsInCompaction(
final boolean verifyChecksumsInCompaction) {
setVerifyChecksumsInCompaction(
nativeHandle_, verifyChecksumsInCompaction);
return this;
}
@Override
public boolean verifyChecksumsInCompaction() {
return verifyChecksumsInCompaction(nativeHandle_);
}
@Override
public ColumnFamilyOptions setMaxSequentialSkipInIterations(
final long maxSequentialSkipInIterations) {
@ -586,18 +573,6 @@ public class ColumnFamilyOptions extends RocksObject
return maxSuccessiveMerges(nativeHandle_);
}
@Override
public ColumnFamilyOptions setMinPartialMergeOperands(
final int minPartialMergeOperands) {
setMinPartialMergeOperands(nativeHandle_, minPartialMergeOperands);
return this;
}
@Override
public int minPartialMergeOperands() {
return minPartialMergeOperands(nativeHandle_);
}
@Override
public ColumnFamilyOptions setOptimizeFiltersForHits(
final boolean optimizeFiltersForHits) {
@ -802,9 +777,6 @@ public class ColumnFamilyOptions extends RocksObject
private native void setPurgeRedundantKvsWhileFlush(
long handle, boolean purgeRedundantKvsWhileFlush);
private native boolean purgeRedundantKvsWhileFlush(long handle);
private native void setVerifyChecksumsInCompaction(
long handle, boolean verifyChecksumsInCompaction);
private native boolean verifyChecksumsInCompaction(long handle);
private native void setMaxSequentialSkipInIterations(
long handle, long maxSequentialSkipInIterations);
private native long maxSequentialSkipInIterations(long handle);
@ -829,9 +801,6 @@ public class ColumnFamilyOptions extends RocksObject
long handle, long maxSuccessiveMerges)
throws IllegalArgumentException;
private native long maxSuccessiveMerges(long handle);
private native void setMinPartialMergeOperands(
long handle, int minPartialMergeOperands);
private native int minPartialMergeOperands(long handle);
private native void setOptimizeFiltersForHits(long handle,
boolean optimizeFiltersForHits);
private native boolean optimizeFiltersForHits(long handle);

27
java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java
Unescape View File

@ -699,33 +699,6 @@ public interface ColumnFamilyOptionsInterface {
*/
int bloomLocality();
/**
* The number of partial merge operands to accumulate before partial
* merge will be performed. Partial merge will not be called
* if the list of values to merge is less than min_partial_merge_operands.
*
* If min_partial_merge_operands &lt; 2, then it will be treated as 2.
*
* Default: 2
*
* @param minPartialMergeOperands min partial merge operands
* @return the reference to the current option.
*/
Object setMinPartialMergeOperands(int minPartialMergeOperands);
/**
* The number of partial merge operands to accumulate before partial
* merge will be performed. Partial merge will not be called
* if the list of values to merge is less than min_partial_merge_operands.
*
* If min_partial_merge_operands &lt; 2, then it will be treated as 2.
*
* Default: 2
*
* @return min partial merge operands
*/
int minPartialMergeOperands();
/**
* <p>This flag specifies that the implementation should optimize the filters
* mainly for cases where keys are found rather than also optimize for keys

15
java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
Unescape View File

@ -148,8 +148,7 @@ public class MutableColumnFamilyOptions {
target_file_size_multiplier(ValueType.INT),
max_bytes_for_level_base(ValueType.LONG),
max_bytes_for_level_multiplier(ValueType.INT),
max_bytes_for_level_multiplier_additional(ValueType.INT_ARRAY),
verify_checksums_in_compaction(ValueType.BOOLEAN);
max_bytes_for_level_multiplier_additional(ValueType.INT_ARRAY);
private final ValueType valueType;
CompactionOption(final ValueType valueType) {
@ -866,18 +865,6 @@ public class MutableColumnFamilyOptions {
CompactionOption.max_bytes_for_level_multiplier_additional);
}
@Override
public MutableColumnFamilyOptionsBuilder setVerifyChecksumsInCompaction(
final boolean verifyChecksumsInCompaction) {
return setBoolean(CompactionOption.verify_checksums_in_compaction,
verifyChecksumsInCompaction);
}
@Override
public boolean verifyChecksumsInCompaction() {
return getBoolean(CompactionOption.verify_checksums_in_compaction);
}
@Override
public MutableColumnFamilyOptionsBuilder setMaxSequentialSkipInIterations(
final long maxSequentialSkipInIterations) {

20
java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java
Unescape View File

@ -543,26 +543,6 @@ public interface MutableColumnFamilyOptionsInterface {
*/
int[] maxBytesForLevelMultiplierAdditional();
/**
* If true, compaction will verify checksum on every read that happens
* as part of compaction
* Default: true
*
* @param verifyChecksumsInCompaction true if compaction verifies
* checksum on every read.
* @return the reference to the current option.
*/
MutableColumnFamilyOptionsInterface setVerifyChecksumsInCompaction(
boolean verifyChecksumsInCompaction);
/**
* If true, compaction will verify checksum on every read that happens
* as part of compaction
* Default: true
*
* @return true if compaction verifies checksum on every read.
*/
boolean verifyChecksumsInCompaction();
/**
* An iteration-&gt;Next() sequentially skips over keys with the same

31
java/src/main/java/org/rocksdb/Options.java
Unescape View File

@ -997,19 +997,6 @@ public class Options extends RocksObject
return this;
}
@Override
public boolean verifyChecksumsInCompaction() {
return verifyChecksumsInCompaction(nativeHandle_);
}
@Override
public Options setVerifyChecksumsInCompaction(
final boolean verifyChecksumsInCompaction) {
setVerifyChecksumsInCompaction(
nativeHandle_, verifyChecksumsInCompaction);
return this;
}
@Override
public long maxSequentialSkipInIterations() {
return maxSequentialSkipInIterations(nativeHandle_);
@ -1092,18 +1079,6 @@ public class Options extends RocksObject
return this;
}
@Override
public int minPartialMergeOperands() {
return minPartialMergeOperands(nativeHandle_);
}
@Override
public Options setMinPartialMergeOperands(
final int minPartialMergeOperands) {
setMinPartialMergeOperands(nativeHandle_, minPartialMergeOperands);
return this;
}
@Override
public Options setOptimizeFiltersForHits(
final boolean optimizeFiltersForHits) {
@ -1406,9 +1381,6 @@ public class Options extends RocksObject
private native void setPurgeRedundantKvsWhileFlush(
long handle, boolean purgeRedundantKvsWhileFlush);
private native boolean purgeRedundantKvsWhileFlush(long handle);
private native void setVerifyChecksumsInCompaction(
long handle, boolean verifyChecksumsInCompaction);
private native boolean verifyChecksumsInCompaction(long handle);
private native void setMaxSequentialSkipInIterations(
long handle, long maxSequentialSkipInIterations);
private native long maxSequentialSkipInIterations(long handle);
@ -1433,9 +1405,6 @@ public class Options extends RocksObject
long handle, long maxSuccessiveMerges)
throws IllegalArgumentException;
private native long maxSuccessiveMerges(long handle);
private native void setMinPartialMergeOperands(
long handle, int minPartialMergeOperands);
private native int minPartialMergeOperands(long handle);
private native void setOptimizeFiltersForHits(long handle,
boolean optimizeFiltersForHits);
private native boolean optimizeFiltersForHits(long handle);

18
java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
Unescape View File

@ -297,15 +297,6 @@ public class ColumnFamilyOptionsTest {
}
}
@Test
public void verifyChecksumsInCompaction() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final boolean boolValue = rand.nextBoolean();
opt.setVerifyChecksumsInCompaction(boolValue);
assertThat(opt.verifyChecksumsInCompaction()).isEqualTo(boolValue);
}
}
@Test
public void maxSequentialSkipInIterations() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
@ -369,15 +360,6 @@ public class ColumnFamilyOptionsTest {
}
}
@Test
public void minPartialMergeOperands() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final int intValue = rand.nextInt();
opt.setMinPartialMergeOperands(intValue);
assertThat(opt.minPartialMergeOperands()).isEqualTo(intValue);
}
}
@Test
public void optimizeFiltersForHits() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {

10
java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java
Unescape View File

@ -21,13 +21,11 @@ public class MutableColumnFamilyOptionsTest {
.setWriteBufferSize(10)
.setInplaceUpdateNumLocks(5)
.setDisableAutoCompactions(true)
.setVerifyChecksumsInCompaction(false)
.setParanoidFileChecks(true);
assertThat(builder.writeBufferSize()).isEqualTo(10);
assertThat(builder.inplaceUpdateNumLocks()).isEqualTo(5);
assertThat(builder.disableAutoCompactions()).isEqualTo(true);
assertThat(builder.verifyChecksumsInCompaction()).isEqualTo(false);
assertThat(builder.paranoidFileChecks()).isEqualTo(true);
}
@ -66,21 +64,18 @@ public class MutableColumnFamilyOptionsTest {
.setWriteBufferSize(10)
.setInplaceUpdateNumLocks(5)
.setDisableAutoCompactions(true)
.setVerifyChecksumsInCompaction(false)
.setParanoidFileChecks(true)
.build()
.toString();
assertThat(str).isEqualTo("write_buffer_size=10;inplace_update_num_locks=5;"
+ "disable_auto_compactions=true;verify_checksums_in_compaction=false;"
+ "paranoid_file_checks=true");
+ "disable_auto_compactions=true;paranoid_file_checks=true");
}
@Test
public void mutableColumnFamilyOptions_parse() {
final String str = "write_buffer_size=10;inplace_update_num_locks=5;"
+ "disable_auto_compactions=true;verify_checksums_in_compaction=false;"
+ "paranoid_file_checks=true";
+ "disable_auto_compactions=true;paranoid_file_checks=true";
final MutableColumnFamilyOptionsBuilder builder =
MutableColumnFamilyOptions.parse(str);
@ -88,7 +83,6 @@ public class MutableColumnFamilyOptionsTest {
assertThat(builder.writeBufferSize()).isEqualTo(10);
assertThat(builder.inplaceUpdateNumLocks()).isEqualTo(5);
assertThat(builder.disableAutoCompactions()).isEqualTo(true);
assertThat(builder.verifyChecksumsInCompaction()).isEqualTo(false);
assertThat(builder.paranoidFileChecks()).isEqualTo(true);
}
}

18
java/src/test/java/org/rocksdb/OptionsTest.java
Unescape View File

@ -260,15 +260,6 @@ public class OptionsTest {
}
}
@Test
public void verifyChecksumsInCompaction() {
try (final Options opt = new Options()) {
final boolean boolValue = rand.nextBoolean();
opt.setVerifyChecksumsInCompaction(boolValue);
assertThat(opt.verifyChecksumsInCompaction()).isEqualTo(boolValue);
}
}
@Test
public void maxSequentialSkipInIterations() {
try (final Options opt = new Options()) {
@ -332,15 +323,6 @@ public class OptionsTest {
}
}
@Test
public void minPartialMergeOperands() {
try (final Options opt = new Options()) {
final int intValue = rand.nextInt();
opt.setMinPartialMergeOperands(intValue);
assertThat(opt.minPartialMergeOperands()).isEqualTo(intValue);
}
}
@Test
public void optimizeFiltersForHits() {
try (final Options opt = new Options()) {

4
util/cf_options.cc
Unescape View File

@ -158,8 +158,6 @@ void MutableCFOptions::Dump(Logger* log) const {
}
Log(log, "max_bytes_for_level_multiplier_additional: %s", result.c_str());
Log(log, " verify_checksums_in_compaction: %d",
verify_checksums_in_compaction);
Log(log, " max_sequential_skip_in_iterations: %" PRIu64,
max_sequential_skip_in_iterations);
Log(log, " paranoid_file_checks: %d",
@ -167,8 +165,6 @@ void MutableCFOptions::Dump(Logger* log) const {
Log(log, " report_bg_io_stats: %d", report_bg_io_stats);
Log(log, " compression: %d",
static_cast<int>(compression));
Log(log, " min_partial_merge_operands: %" PRIu32,
min_partial_merge_operands);
}
} // namespace rocksdb

10
util/cf_options.h
Unescape View File

@ -145,13 +145,11 @@ struct MutableCFOptions {
max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
max_bytes_for_level_multiplier_additional(
options.max_bytes_for_level_multiplier_additional),
verify_checksums_in_compaction(options.verify_checksums_in_compaction),
max_sequential_skip_in_iterations(
options.max_sequential_skip_in_iterations),
paranoid_file_checks(options.paranoid_file_checks),
report_bg_io_stats(options.report_bg_io_stats),
compression(options.compression),
min_partial_merge_operands(options.min_partial_merge_operands) {
compression(options.compression) {
RefreshDerivedOptions(options.num_levels, options.compaction_style);
}
@ -174,12 +172,10 @@ struct MutableCFOptions {
target_file_size_multiplier(0),
max_bytes_for_level_base(0),
max_bytes_for_level_multiplier(0),
verify_checksums_in_compaction(false),
max_sequential_skip_in_iterations(0),
paranoid_file_checks(false),
report_bg_io_stats(false),
compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
min_partial_merge_operands(2) {}
compression(Snappy_Supported() ? kSnappyCompression : kNoCompression) {}
// Must be called after any change to MutableCFOptions
void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style);
@ -222,14 +218,12 @@ struct MutableCFOptions {
uint64_t max_bytes_for_level_base;
double max_bytes_for_level_multiplier;
std::vector<int> max_bytes_for_level_multiplier_additional;
bool verify_checksums_in_compaction;
// Misc options
uint64_t max_sequential_skip_in_iterations;
bool paranoid_file_checks;
bool report_bg_io_stats;
CompressionType compression;
uint32_t min_partial_merge_operands;
// Derived options
// Per-level target file size.

6
util/options.cc
Unescape View File

@ -85,7 +85,6 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush),
compaction_style(options.compaction_style),
compaction_pri(options.compaction_pri),
verify_checksums_in_compaction(options.verify_checksums_in_compaction),
compaction_options_universal(options.compaction_options_universal),
compaction_options_fifo(options.compaction_options_fifo),
max_sequential_skip_in_iterations(
@ -104,7 +103,6 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
options.memtable_insert_with_hint_prefix_extractor),
bloom_locality(options.bloom_locality),
max_successive_merges(options.max_successive_merges),
min_partial_merge_operands(options.min_partial_merge_operands),
optimize_filters_for_hits(options.optimize_filters_for_hits),
paranoid_file_checks(options.paranoid_file_checks),
force_consistency_checks(options.force_consistency_checks),
@ -282,8 +280,6 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
rate_limit_delay_max_milliseconds);
Header(log, " Options.disable_auto_compactions: %d",
disable_auto_compactions);
Header(log, " Options.verify_checksums_in_compaction: %d",
verify_checksums_in_compaction);
const auto& it_compaction_style =
compaction_style_to_string.find(compaction_style);
@ -335,8 +331,6 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
Header(log,
" Options.inplace_update_num_locks: %" ROCKSDB_PRIszt,
inplace_update_num_locks);
Header(log, " Options.min_partial_merge_operands: %u",
min_partial_merge_operands);
// TODO: easier config for bloom (maybe based on avg key/value size)
Header(log, " Options.memtable_prefix_bloom_size_ratio: %f",
memtable_prefix_bloom_size_ratio);

5
util/options_helper.cc
Unescape View File

@ -163,17 +163,12 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value);
}
cf_opts.verify_checksums_in_compaction =
mutable_cf_options.verify_checksums_in_compaction;
// Misc options
cf_opts.max_sequential_skip_in_iterations =
mutable_cf_options.max_sequential_skip_in_iterations;
cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks;
cf_opts.report_bg_io_stats = mutable_cf_options.report_bg_io_stats;
cf_opts.compression = mutable_cf_options.compression;
cf_opts.min_partial_merge_operands =
mutable_cf_options.min_partial_merge_operands;
cf_opts.table_factory = options.table_factory;
// TODO(yhchiang): find some way to handle the following derived options

8
util/options_helper.h
Unescape View File

@ -411,9 +411,7 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{offsetof(struct ColumnFamilyOptions, purge_redundant_kvs_while_flush),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"verify_checksums_in_compaction",
{offsetof(struct ColumnFamilyOptions, verify_checksums_in_compaction),
OptionType::kBoolean, OptionVerificationType::kNormal, true,
offsetof(struct MutableCFOptions, verify_checksums_in_compaction)}},
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, 0}},
{"soft_pending_compaction_bytes_limit",
{offsetof(struct ColumnFamilyOptions, soft_pending_compaction_bytes_limit),
OptionType::kUInt64T, OptionVerificationType::kNormal, true,
@ -501,9 +499,7 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{"memtable_prefix_bloom_probes",
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, 0}},
{"min_partial_merge_operands",
{offsetof(struct ColumnFamilyOptions, min_partial_merge_operands),
OptionType::kUInt32T, OptionVerificationType::kNormal, true,
offsetof(struct MutableCFOptions, min_partial_merge_operands)}},
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, 0}},
{"max_bytes_for_level_base",
{offsetof(struct ColumnFamilyOptions, max_bytes_for_level_base),
OptionType::kUInt64T, OptionVerificationType::kNormal, true,

2
util/options_settable_test.cc
Unescape View File

@ -411,7 +411,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"memtable_factory=SkipListFactory;"
"compression=kNoCompression;"
"bottommost_compression=kDisableCompressionOption;"
"min_partial_merge_operands=7576;"
"level0_stop_writes_trigger=33;"
"num_levels=99;"
"level0_slowdown_writes_trigger=22;"
@ -420,7 +419,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"soft_rate_limit=530.615385;"
"soft_pending_compaction_bytes_limit=0;"
"max_write_buffer_number_to_maintain=84;"
"verify_checksums_in_compaction=false;"
"merge_operator=aabcxehazrMergeOperator;"
"memtable_prefix_bloom_size_ratio=0.4642;"
"memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;"

2
util/options_test.cc
Unescape View File

@ -174,7 +174,6 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
ASSERT_EQ(new_cf_opt.verify_checksums_in_compaction, false);
ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
static_cast<uint64_t>(23));
ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
@ -185,7 +184,6 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U);
ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
ASSERT_EQ(new_cf_opt.min_partial_merge_operands, 31U);
ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()),

2
util/testutil.cc
Unescape View File

@ -303,7 +303,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
cf_opt->optimize_filters_for_hits = rnd->Uniform(2);
cf_opt->paranoid_file_checks = rnd->Uniform(2);
cf_opt->purge_redundant_kvs_while_flush = rnd->Uniform(2);
cf_opt->verify_checksums_in_compaction = rnd->Uniform(2);
cf_opt->force_consistency_checks = rnd->Uniform(2);
// double options
@ -339,7 +338,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
// uint32_t options
cf_opt->bloom_locality = rnd->Uniform(10000);
cf_opt->min_partial_merge_operands = rnd->Uniform(10000);
cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);
// uint64_t options

Write Preview
Loading…
Cancel
Save