[rocksdb][PR] Remove option min_partial_merge_operands and verify_checksums_in_comp…

Summary:
…action

 The two options, min_partial_merge_operands and verify_checksums_in_compaction, are not seldom used. Remove them to reduce the total number of options. Also remove them from Java and C interface.
Closes https://github.com/facebook/rocksdb/pull/1902

Differential Revision: D4601219

Pulled By: siying

fbshipit-source-id: aad4cb2
main
Siying Dong 8 years ago committed by Facebook Github Bot
parent 1ba2804b7f
commit 8efb5ffa2a
  1. 2
      HISTORY.md
  2. 1
      db/builder.cc
  3. 10
      db/c.cc
  4. 2
      db/compaction_iterator_test.cc
  5. 1
      db/compaction_job.cc
  6. 6
      db/db_test.cc
  7. 3
      db/merge_helper.cc
  8. 3
      db/merge_helper.h
  9. 2
      db/merge_helper_test.cc
  10. 32
      db/merge_test.cc
  11. 3
      db/version_set.cc
  12. 5
      include/rocksdb/c.h
  13. 6
      include/rocksdb/merge_operator.h
  14. 16
      include/rocksdb/options.h
  15. 96
      java/rocksjni/options.cc
  16. 31
      java/src/main/java/org/rocksdb/ColumnFamilyOptions.java
  17. 27
      java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java
  18. 15
      java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java
  19. 20
      java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java
  20. 31
      java/src/main/java/org/rocksdb/Options.java
  21. 18
      java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java
  22. 10
      java/src/test/java/org/rocksdb/MutableColumnFamilyOptionsTest.java
  23. 18
      java/src/test/java/org/rocksdb/OptionsTest.java
  24. 4
      util/cf_options.cc
  25. 10
      util/cf_options.h
  26. 6
      util/options.cc
  27. 5
      util/options_helper.cc
  28. 8
      util/options_helper.h
  29. 2
      util/options_settable_test.cc
  30. 2
      util/options_test.cc
  31. 2
      util/testutil.cc

@ -3,6 +3,8 @@
### Public API Change
* Remove disableDataSync option.
* Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0.
* Remove option min_partial_merge_operands. Partial merge operands will always be merged in flush or compaction if there are more than one.
* Remove option verify_checksums_in_compaction. Compaction will always verify checksum.
## 5.2.0 (02/08/2017)
### Public API Change

@ -122,7 +122,6 @@ Status BuildTable(
MergeHelper merge(env, internal_comparator.user_comparator(),
ioptions.merge_operator, nullptr, ioptions.info_log,
mutable_cf_options.min_partial_merge_operands,
true /* internal key corruption is not ok */,
snapshots.empty() ? 0 : snapshots.back());

@ -1824,11 +1824,6 @@ void rocksdb_options_set_enable_write_thread_adaptive_yield(
opt->rep.enable_write_thread_adaptive_yield = v;
}
void rocksdb_options_set_verify_checksums_in_compaction(
rocksdb_options_t* opt, unsigned char v) {
opt->rep.verify_checksums_in_compaction = v;
}
void rocksdb_options_set_max_sequential_skip_in_iterations(
rocksdb_options_t* opt, uint64_t v) {
opt->rep.max_sequential_skip_in_iterations = v;
@ -1980,11 +1975,6 @@ void rocksdb_options_set_max_successive_merges(
opt->rep.max_successive_merges = v;
}
void rocksdb_options_set_min_partial_merge_operands(
rocksdb_options_t* opt, uint32_t v) {
opt->rep.min_partial_merge_operands = v;
}
void rocksdb_options_set_bloom_locality(
rocksdb_options_t* opt, uint32_t v) {
opt->rep.bloom_locality = v;

@ -182,7 +182,7 @@ class CompactionIteratorTest : public testing::Test {
}
merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, merge_op, filter,
nullptr, 0U, false, 0, 0, nullptr,
nullptr, false, 0, 0, nullptr,
&shutting_down_));
iter_.reset(new LoggingForwardVectorIterator(ks, vs));
iter_->SeekToFirst();

@ -721,7 +721,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
MergeHelper merge(
env_, cfd->user_comparator(), cfd->ioptions()->merge_operator,
compaction_filter, db_options_.info_log.get(),
mutable_cf_options->min_partial_merge_operands,
false /* internal key corruption is expected */,
existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
compact_->compaction->level(), db_options_.statistics.get(),

@ -4691,12 +4691,6 @@ TEST_F(DBTest, DynamicMiscOptions) {
ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1],
&mutable_cf_options));
ASSERT_EQ(true, mutable_cf_options.report_bg_io_stats);
// Test min_partial_merge_operands
ASSERT_OK(
dbfull()->SetOptions(handles_[1], {{"min_partial_merge_operands", "4"}}));
ASSERT_OK(dbfull()->TEST_GetLatestMutableCFOptions(handles_[1],
&mutable_cf_options));
ASSERT_EQ(4, mutable_cf_options.min_partial_merge_operands);
// Test compression
// sanity check
ASSERT_OK(dbfull()->SetOptions({{"compression", "kNoCompression"}}));

@ -283,8 +283,7 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
// Attempt to use the user's associative merge function to
// merge the stacked merge operands into a single operand.
s = Status::MergeInProgress();
if (merge_context_.GetNumOperands() >= 2 &&
merge_context_.GetNumOperands() >= min_partial_merge_operands_) {
if (merge_context_.GetNumOperands() >= 2) {
bool merge_success = false;
std::string merge_result;
{

@ -32,7 +32,6 @@ class MergeHelper {
MergeHelper(Env* env, const Comparator* user_comparator,
const MergeOperator* user_merge_operator,
const CompactionFilter* compaction_filter, Logger* logger,
unsigned min_partial_merge_operands,
bool assert_valid_internal_key, SequenceNumber latest_snapshot,
int level = 0, Statistics* stats = nullptr,
const std::atomic<bool>* shutting_down = nullptr)
@ -42,7 +41,6 @@ class MergeHelper {
compaction_filter_(compaction_filter),
shutting_down_(shutting_down),
logger_(logger),
min_partial_merge_operands_(min_partial_merge_operands),
assert_valid_internal_key_(assert_valid_internal_key),
latest_snapshot_(latest_snapshot),
level_(level),
@ -156,7 +154,6 @@ class MergeHelper {
const CompactionFilter* compaction_filter_;
const std::atomic<bool>* shutting_down_;
Logger* logger_;
unsigned min_partial_merge_operands_;
bool assert_valid_internal_key_; // enforce no internal key corruption?
SequenceNumber latest_snapshot_;
int level_;

@ -28,7 +28,7 @@ class MergeHelperTest : public testing::Test {
iter_->SeekToFirst();
merge_helper_.reset(new MergeHelper(env_, BytewiseComparator(),
merge_op_.get(), filter_.get(), nullptr,
2U, false, latest_snapshot));
false, latest_snapshot));
return merge_helper_->MergeUntil(iter_.get(), nullptr /* range_del_agg */,
stop_before, at_bottom);
}

@ -76,14 +76,12 @@ class CountMergeOperator : public AssociativeMergeOperator {
namespace {
std::shared_ptr<DB> OpenDb(const std::string& dbname, const bool ttl = false,
const size_t max_successive_merges = 0,
const uint32_t min_partial_merge_operands = 2) {
const size_t max_successive_merges = 0) {
DB* db;
Options options;
options.create_if_missing = true;
options.merge_operator = std::make_shared<CountMergeOperator>();
options.max_successive_merges = max_successive_merges;
options.min_partial_merge_operands = min_partial_merge_operands;
Status s;
DestroyDB(dbname, Options());
// DBWithTTL is not supported in ROCKSDB_LITE
@ -448,20 +446,20 @@ void runTest(int argc, const std::string& dbname, const bool use_ttl = false) {
{
std::cout << "Test Partial-Merge\n";
size_t max_merge = 100;
for (uint32_t min_merge = 5; min_merge < 25; min_merge += 5) {
for (uint32_t count = min_merge - 1; count <= min_merge + 1; count++) {
auto db = OpenDb(dbname, use_ttl, max_merge, min_merge);
MergeBasedCounters counters(db, 0);
testPartialMerge(&counters, db.get(), max_merge, min_merge, count);
DestroyDB(dbname, Options());
}
{
auto db = OpenDb(dbname, use_ttl, max_merge, min_merge);
MergeBasedCounters counters(db, 0);
testPartialMerge(&counters, db.get(), max_merge, min_merge,
min_merge * 10);
DestroyDB(dbname, Options());
}
// Min merge is hard-coded to 2.
uint32_t min_merge = 2;
for (uint32_t count = min_merge - 1; count <= min_merge + 1; count++) {
auto db = OpenDb(dbname, use_ttl, max_merge);
MergeBasedCounters counters(db, 0);
testPartialMerge(&counters, db.get(), max_merge, min_merge, count);
DestroyDB(dbname, Options());
}
{
auto db = OpenDb(dbname, use_ttl, max_merge);
MergeBasedCounters counters(db, 0);
testPartialMerge(&counters, db.get(), max_merge, min_merge,
min_merge * 10);
DestroyDB(dbname, Options());
}
}

@ -3431,8 +3431,7 @@ InternalIterator* VersionSet::MakeInputIterator(
const Compaction* c, RangeDelAggregator* range_del_agg) {
auto cfd = c->column_family_data();
ReadOptions read_options;
read_options.verify_checksums =
c->mutable_cf_options()->verify_checksums_in_compaction;
read_options.verify_checksums = true;
read_options.fill_cache = false;
if (c->ShouldFormSubcompactions()) {
read_options.total_order_seek = true;

@ -700,9 +700,6 @@ extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_enable_write_thread_adaptive_yield(rocksdb_options_t*,
unsigned char);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_verify_checksums_in_compaction(rocksdb_options_t*,
unsigned char);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_max_sequential_skip_in_iterations(rocksdb_options_t*,
uint64_t);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_disable_auto_compactions(
@ -735,8 +732,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_memtable_huge_page_size(
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_successive_merges(
rocksdb_options_t*, size_t);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_min_partial_merge_operands(
rocksdb_options_t*, uint32_t);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bloom_locality(
rocksdb_options_t*, uint32_t);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_inplace_update_support(

@ -165,10 +165,8 @@ class MergeOperator {
//
// The string that new_value is pointing to will be empty.
//
// The PartialMergeMulti function will be called only when the list of
// operands are long enough. The minimum amount of operands that will be
// passed to the function are specified by the "min_partial_merge_operands"
// option.
// The PartialMergeMulti function will be called when there are at least two
// operands.
//
// In the default implementation, PartialMergeMulti will invoke PartialMerge
// multiple times, where each time it only merges two operands. Developers

@ -615,13 +615,6 @@ struct ColumnFamilyOptions {
// Default: kByCompensatedSize
CompactionPri compaction_pri = kByCompensatedSize;
// If true, compaction will verify checksum on every read that happens
// as part of compaction
//
// Default: true
//
// Dynamically changeable through SetOptions() API
bool verify_checksums_in_compaction = true;
// The options needed to support Universal Style compactions
CompactionOptionsUniversal compaction_options_universal;
@ -798,15 +791,6 @@ struct ColumnFamilyOptions {
// Dynamically changeable through SetOptions() API
size_t max_successive_merges = 0;
// The number of partial merge operands to accumulate before partial
// merge will be performed. Partial merge will not be called
// if the list of values to merge is less than min_partial_merge_operands.
//
// If min_partial_merge_operands < 2, then it will be treated as 2.
//
// Default: 2
uint32_t min_partial_merge_operands = 2;
// This flag specifies that the implementation should optimize the filters
// mainly for cases where keys are found rather than also optimize for keys
// missed. This would be used in cases where the application knows that

@ -1707,30 +1707,6 @@ void Java_org_rocksdb_Options_setPurgeRedundantKvsWhileFlush(
static_cast<bool>(jpurge_redundant_kvs_while_flush);
}
/*
* Class: org_rocksdb_Options
* Method: verifyChecksumsInCompaction
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_Options_verifyChecksumsInCompaction(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->verify_checksums_in_compaction;
}
/*
* Class: org_rocksdb_Options
* Method: setVerifyChecksumsInCompaction
* Signature: (JZ)V
*/
void Java_org_rocksdb_Options_setVerifyChecksumsInCompaction(
JNIEnv* env, jobject jobj, jlong jhandle,
jboolean jverify_checksums_in_compaction) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->verify_checksums_in_compaction =
static_cast<bool>(jverify_checksums_in_compaction);
}
/*
* Class: org_rocksdb_Options
* Method: maxSequentialSkipInIterations
@ -1882,30 +1858,6 @@ void Java_org_rocksdb_Options_setMaxSuccessiveMerges(
}
}
/*
* Class: org_rocksdb_Options
* Method: minPartialMergeOperands
* Signature: (J)I
*/
jint Java_org_rocksdb_Options_minPartialMergeOperands(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::Options*>(
jhandle)->min_partial_merge_operands;
}
/*
* Class: org_rocksdb_Options
* Method: setMinPartialMergeOperands
* Signature: (JI)V
*/
void Java_org_rocksdb_Options_setMinPartialMergeOperands(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmin_partial_merge_operands) {
reinterpret_cast<rocksdb::Options*>(
jhandle)->min_partial_merge_operands =
static_cast<int32_t>(jmin_partial_merge_operands);
}
/*
* Class: org_rocksdb_Options
* Method: optimizeFiltersForHits
@ -3013,30 +2965,6 @@ void Java_org_rocksdb_ColumnFamilyOptions_setPurgeRedundantKvsWhileFlush(
static_cast<bool>(jpurge_redundant_kvs_while_flush);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: verifyChecksumsInCompaction
* Signature: (J)Z
*/
jboolean Java_org_rocksdb_ColumnFamilyOptions_verifyChecksumsInCompaction(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->verify_checksums_in_compaction;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setVerifyChecksumsInCompaction
* Signature: (JZ)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setVerifyChecksumsInCompaction(
JNIEnv* env, jobject jobj, jlong jhandle,
jboolean jverify_checksums_in_compaction) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->verify_checksums_in_compaction =
static_cast<bool>(jverify_checksums_in_compaction);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: maxSequentialSkipInIterations
@ -3189,30 +3117,6 @@ void Java_org_rocksdb_ColumnFamilyOptions_setMaxSuccessiveMerges(
}
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: minPartialMergeOperands
* Signature: (J)I
*/
jint Java_org_rocksdb_ColumnFamilyOptions_minPartialMergeOperands(
JNIEnv* env, jobject jobj, jlong jhandle) {
return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->min_partial_merge_operands;
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: setMinPartialMergeOperands
* Signature: (JI)V
*/
void Java_org_rocksdb_ColumnFamilyOptions_setMinPartialMergeOperands(
JNIEnv* env, jobject jobj, jlong jhandle,
jint jmin_partial_merge_operands) {
reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
jhandle)->min_partial_merge_operands =
static_cast<int32_t>(jmin_partial_merge_operands);
}
/*
* Class: org_rocksdb_ColumnFamilyOptions
* Method: optimizeFiltersForHits

@ -473,19 +473,6 @@ public class ColumnFamilyOptions extends RocksObject
return maxTableFilesSizeFIFO(nativeHandle_);
}
@Override
public ColumnFamilyOptions setVerifyChecksumsInCompaction(
final boolean verifyChecksumsInCompaction) {
setVerifyChecksumsInCompaction(
nativeHandle_, verifyChecksumsInCompaction);
return this;
}
@Override
public boolean verifyChecksumsInCompaction() {
return verifyChecksumsInCompaction(nativeHandle_);
}
@Override
public ColumnFamilyOptions setMaxSequentialSkipInIterations(
final long maxSequentialSkipInIterations) {
@ -586,18 +573,6 @@ public class ColumnFamilyOptions extends RocksObject
return maxSuccessiveMerges(nativeHandle_);
}
@Override
public ColumnFamilyOptions setMinPartialMergeOperands(
final int minPartialMergeOperands) {
setMinPartialMergeOperands(nativeHandle_, minPartialMergeOperands);
return this;
}
@Override
public int minPartialMergeOperands() {
return minPartialMergeOperands(nativeHandle_);
}
@Override
public ColumnFamilyOptions setOptimizeFiltersForHits(
final boolean optimizeFiltersForHits) {
@ -802,9 +777,6 @@ public class ColumnFamilyOptions extends RocksObject
private native void setPurgeRedundantKvsWhileFlush(
long handle, boolean purgeRedundantKvsWhileFlush);
private native boolean purgeRedundantKvsWhileFlush(long handle);
private native void setVerifyChecksumsInCompaction(
long handle, boolean verifyChecksumsInCompaction);
private native boolean verifyChecksumsInCompaction(long handle);
private native void setMaxSequentialSkipInIterations(
long handle, long maxSequentialSkipInIterations);
private native long maxSequentialSkipInIterations(long handle);
@ -829,9 +801,6 @@ public class ColumnFamilyOptions extends RocksObject
long handle, long maxSuccessiveMerges)
throws IllegalArgumentException;
private native long maxSuccessiveMerges(long handle);
private native void setMinPartialMergeOperands(
long handle, int minPartialMergeOperands);
private native int minPartialMergeOperands(long handle);
private native void setOptimizeFiltersForHits(long handle,
boolean optimizeFiltersForHits);
private native boolean optimizeFiltersForHits(long handle);

@ -699,33 +699,6 @@ public interface ColumnFamilyOptionsInterface {
*/
int bloomLocality();
/**
* The number of partial merge operands to accumulate before partial
* merge will be performed. Partial merge will not be called
* if the list of values to merge is less than min_partial_merge_operands.
*
* If min_partial_merge_operands &lt; 2, then it will be treated as 2.
*
* Default: 2
*
* @param minPartialMergeOperands min partial merge operands
* @return the reference to the current option.
*/
Object setMinPartialMergeOperands(int minPartialMergeOperands);
/**
* The number of partial merge operands to accumulate before partial
* merge will be performed. Partial merge will not be called
* if the list of values to merge is less than min_partial_merge_operands.
*
* If min_partial_merge_operands &lt; 2, then it will be treated as 2.
*
* Default: 2
*
* @return min partial merge operands
*/
int minPartialMergeOperands();
/**
* <p>This flag specifies that the implementation should optimize the filters
* mainly for cases where keys are found rather than also optimize for keys

@ -148,8 +148,7 @@ public class MutableColumnFamilyOptions {
target_file_size_multiplier(ValueType.INT),
max_bytes_for_level_base(ValueType.LONG),
max_bytes_for_level_multiplier(ValueType.INT),
max_bytes_for_level_multiplier_additional(ValueType.INT_ARRAY),
verify_checksums_in_compaction(ValueType.BOOLEAN);
max_bytes_for_level_multiplier_additional(ValueType.INT_ARRAY);
private final ValueType valueType;
CompactionOption(final ValueType valueType) {
@ -866,18 +865,6 @@ public class MutableColumnFamilyOptions {
CompactionOption.max_bytes_for_level_multiplier_additional);
}
@Override
public MutableColumnFamilyOptionsBuilder setVerifyChecksumsInCompaction(
final boolean verifyChecksumsInCompaction) {
return setBoolean(CompactionOption.verify_checksums_in_compaction,
verifyChecksumsInCompaction);
}
@Override
public boolean verifyChecksumsInCompaction() {
return getBoolean(CompactionOption.verify_checksums_in_compaction);
}
@Override
public MutableColumnFamilyOptionsBuilder setMaxSequentialSkipInIterations(
final long maxSequentialSkipInIterations) {

@ -543,26 +543,6 @@ public interface MutableColumnFamilyOptionsInterface {
*/
int[] maxBytesForLevelMultiplierAdditional();
/**
* If true, compaction will verify checksum on every read that happens
* as part of compaction
* Default: true
*
* @param verifyChecksumsInCompaction true if compaction verifies
* checksum on every read.
* @return the reference to the current option.
*/
MutableColumnFamilyOptionsInterface setVerifyChecksumsInCompaction(
boolean verifyChecksumsInCompaction);
/**
* If true, compaction will verify checksum on every read that happens
* as part of compaction
* Default: true
*
* @return true if compaction verifies checksum on every read.
*/
boolean verifyChecksumsInCompaction();
/**
* An iteration-&gt;Next() sequentially skips over keys with the same

@ -997,19 +997,6 @@ public class Options extends RocksObject
return this;
}
@Override
public boolean verifyChecksumsInCompaction() {
return verifyChecksumsInCompaction(nativeHandle_);
}
@Override
public Options setVerifyChecksumsInCompaction(
final boolean verifyChecksumsInCompaction) {
setVerifyChecksumsInCompaction(
nativeHandle_, verifyChecksumsInCompaction);
return this;
}
@Override
public long maxSequentialSkipInIterations() {
return maxSequentialSkipInIterations(nativeHandle_);
@ -1092,18 +1079,6 @@ public class Options extends RocksObject
return this;
}
@Override
public int minPartialMergeOperands() {
return minPartialMergeOperands(nativeHandle_);
}
@Override
public Options setMinPartialMergeOperands(
final int minPartialMergeOperands) {
setMinPartialMergeOperands(nativeHandle_, minPartialMergeOperands);
return this;
}
@Override
public Options setOptimizeFiltersForHits(
final boolean optimizeFiltersForHits) {
@ -1406,9 +1381,6 @@ public class Options extends RocksObject
private native void setPurgeRedundantKvsWhileFlush(
long handle, boolean purgeRedundantKvsWhileFlush);
private native boolean purgeRedundantKvsWhileFlush(long handle);
private native void setVerifyChecksumsInCompaction(
long handle, boolean verifyChecksumsInCompaction);
private native boolean verifyChecksumsInCompaction(long handle);
private native void setMaxSequentialSkipInIterations(
long handle, long maxSequentialSkipInIterations);
private native long maxSequentialSkipInIterations(long handle);
@ -1433,9 +1405,6 @@ public class Options extends RocksObject
long handle, long maxSuccessiveMerges)
throws IllegalArgumentException;
private native long maxSuccessiveMerges(long handle);
private native void setMinPartialMergeOperands(
long handle, int minPartialMergeOperands);
private native int minPartialMergeOperands(long handle);
private native void setOptimizeFiltersForHits(long handle,
boolean optimizeFiltersForHits);
private native boolean optimizeFiltersForHits(long handle);

@ -297,15 +297,6 @@ public class ColumnFamilyOptionsTest {
}
}
@Test
public void verifyChecksumsInCompaction() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final boolean boolValue = rand.nextBoolean();
opt.setVerifyChecksumsInCompaction(boolValue);
assertThat(opt.verifyChecksumsInCompaction()).isEqualTo(boolValue);
}
}
@Test
public void maxSequentialSkipInIterations() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
@ -369,15 +360,6 @@ public class ColumnFamilyOptionsTest {
}
}
@Test
public void minPartialMergeOperands() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
final int intValue = rand.nextInt();
opt.setMinPartialMergeOperands(intValue);
assertThat(opt.minPartialMergeOperands()).isEqualTo(intValue);
}
}
@Test
public void optimizeFiltersForHits() {
try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {

@ -21,13 +21,11 @@ public class MutableColumnFamilyOptionsTest {
.setWriteBufferSize(10)
.setInplaceUpdateNumLocks(5)
.setDisableAutoCompactions(true)
.setVerifyChecksumsInCompaction(false)
.setParanoidFileChecks(true);
assertThat(builder.writeBufferSize()).isEqualTo(10);
assertThat(builder.inplaceUpdateNumLocks()).isEqualTo(5);
assertThat(builder.disableAutoCompactions()).isEqualTo(true);
assertThat(builder.verifyChecksumsInCompaction()).isEqualTo(false);
assertThat(builder.paranoidFileChecks()).isEqualTo(true);
}
@ -66,21 +64,18 @@ public class MutableColumnFamilyOptionsTest {
.setWriteBufferSize(10)
.setInplaceUpdateNumLocks(5)
.setDisableAutoCompactions(true)
.setVerifyChecksumsInCompaction(false)
.setParanoidFileChecks(true)
.build()
.toString();
assertThat(str).isEqualTo("write_buffer_size=10;inplace_update_num_locks=5;"
+ "disable_auto_compactions=true;verify_checksums_in_compaction=false;"
+ "paranoid_file_checks=true");
+ "disable_auto_compactions=true;paranoid_file_checks=true");
}
@Test
public void mutableColumnFamilyOptions_parse() {
final String str = "write_buffer_size=10;inplace_update_num_locks=5;"
+ "disable_auto_compactions=true;verify_checksums_in_compaction=false;"
+ "paranoid_file_checks=true";
+ "disable_auto_compactions=true;paranoid_file_checks=true";
final MutableColumnFamilyOptionsBuilder builder =
MutableColumnFamilyOptions.parse(str);
@ -88,7 +83,6 @@ public class MutableColumnFamilyOptionsTest {
assertThat(builder.writeBufferSize()).isEqualTo(10);
assertThat(builder.inplaceUpdateNumLocks()).isEqualTo(5);
assertThat(builder.disableAutoCompactions()).isEqualTo(true);
assertThat(builder.verifyChecksumsInCompaction()).isEqualTo(false);
assertThat(builder.paranoidFileChecks()).isEqualTo(true);
}
}

@ -260,15 +260,6 @@ public class OptionsTest {
}
}
@Test
public void verifyChecksumsInCompaction() {
try (final Options opt = new Options()) {
final boolean boolValue = rand.nextBoolean();
opt.setVerifyChecksumsInCompaction(boolValue);
assertThat(opt.verifyChecksumsInCompaction()).isEqualTo(boolValue);
}
}
@Test
public void maxSequentialSkipInIterations() {
try (final Options opt = new Options()) {
@ -332,15 +323,6 @@ public class OptionsTest {
}
}
@Test
public void minPartialMergeOperands() {
try (final Options opt = new Options()) {
final int intValue = rand.nextInt();
opt.setMinPartialMergeOperands(intValue);
assertThat(opt.minPartialMergeOperands()).isEqualTo(intValue);
}
}
@Test
public void optimizeFiltersForHits() {
try (final Options opt = new Options()) {

@ -158,8 +158,6 @@ void MutableCFOptions::Dump(Logger* log) const {
}
Log(log, "max_bytes_for_level_multiplier_additional: %s", result.c_str());
Log(log, " verify_checksums_in_compaction: %d",
verify_checksums_in_compaction);
Log(log, " max_sequential_skip_in_iterations: %" PRIu64,
max_sequential_skip_in_iterations);
Log(log, " paranoid_file_checks: %d",
@ -167,8 +165,6 @@ void MutableCFOptions::Dump(Logger* log) const {
Log(log, " report_bg_io_stats: %d", report_bg_io_stats);
Log(log, " compression: %d",
static_cast<int>(compression));
Log(log, " min_partial_merge_operands: %" PRIu32,
min_partial_merge_operands);
}
} // namespace rocksdb

@ -145,13 +145,11 @@ struct MutableCFOptions {
max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier),
max_bytes_for_level_multiplier_additional(
options.max_bytes_for_level_multiplier_additional),
verify_checksums_in_compaction(options.verify_checksums_in_compaction),
max_sequential_skip_in_iterations(
options.max_sequential_skip_in_iterations),
paranoid_file_checks(options.paranoid_file_checks),
report_bg_io_stats(options.report_bg_io_stats),
compression(options.compression),
min_partial_merge_operands(options.min_partial_merge_operands) {
compression(options.compression) {
RefreshDerivedOptions(options.num_levels, options.compaction_style);
}
@ -174,12 +172,10 @@ struct MutableCFOptions {
target_file_size_multiplier(0),
max_bytes_for_level_base(0),
max_bytes_for_level_multiplier(0),
verify_checksums_in_compaction(false),
max_sequential_skip_in_iterations(0),
paranoid_file_checks(false),
report_bg_io_stats(false),
compression(Snappy_Supported() ? kSnappyCompression : kNoCompression),
min_partial_merge_operands(2) {}
compression(Snappy_Supported() ? kSnappyCompression : kNoCompression) {}
// Must be called after any change to MutableCFOptions
void RefreshDerivedOptions(int num_levels, CompactionStyle compaction_style);
@ -222,14 +218,12 @@ struct MutableCFOptions {
uint64_t max_bytes_for_level_base;
double max_bytes_for_level_multiplier;
std::vector<int> max_bytes_for_level_multiplier_additional;
bool verify_checksums_in_compaction;
// Misc options
uint64_t max_sequential_skip_in_iterations;
bool paranoid_file_checks;
bool report_bg_io_stats;
CompressionType compression;
uint32_t min_partial_merge_operands;
// Derived options
// Per-level target file size.

@ -85,7 +85,6 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
purge_redundant_kvs_while_flush(options.purge_redundant_kvs_while_flush),
compaction_style(options.compaction_style),
compaction_pri(options.compaction_pri),
verify_checksums_in_compaction(options.verify_checksums_in_compaction),
compaction_options_universal(options.compaction_options_universal),
compaction_options_fifo(options.compaction_options_fifo),
max_sequential_skip_in_iterations(
@ -104,7 +103,6 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options)
options.memtable_insert_with_hint_prefix_extractor),
bloom_locality(options.bloom_locality),
max_successive_merges(options.max_successive_merges),
min_partial_merge_operands(options.min_partial_merge_operands),
optimize_filters_for_hits(options.optimize_filters_for_hits),
paranoid_file_checks(options.paranoid_file_checks),
force_consistency_checks(options.force_consistency_checks),
@ -282,8 +280,6 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
rate_limit_delay_max_milliseconds);
Header(log, " Options.disable_auto_compactions: %d",
disable_auto_compactions);
Header(log, " Options.verify_checksums_in_compaction: %d",
verify_checksums_in_compaction);
const auto& it_compaction_style =
compaction_style_to_string.find(compaction_style);
@ -335,8 +331,6 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
Header(log,
" Options.inplace_update_num_locks: %" ROCKSDB_PRIszt,
inplace_update_num_locks);
Header(log, " Options.min_partial_merge_operands: %u",
min_partial_merge_operands);
// TODO: easier config for bloom (maybe based on avg key/value size)
Header(log, " Options.memtable_prefix_bloom_size_ratio: %f",
memtable_prefix_bloom_size_ratio);

@ -163,17 +163,12 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value);
}
cf_opts.verify_checksums_in_compaction =
mutable_cf_options.verify_checksums_in_compaction;
// Misc options
cf_opts.max_sequential_skip_in_iterations =
mutable_cf_options.max_sequential_skip_in_iterations;
cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks;
cf_opts.report_bg_io_stats = mutable_cf_options.report_bg_io_stats;
cf_opts.compression = mutable_cf_options.compression;
cf_opts.min_partial_merge_operands =
mutable_cf_options.min_partial_merge_operands;
cf_opts.table_factory = options.table_factory;
// TODO(yhchiang): find some way to handle the following derived options

@ -411,9 +411,7 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{offsetof(struct ColumnFamilyOptions, purge_redundant_kvs_while_flush),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"verify_checksums_in_compaction",
{offsetof(struct ColumnFamilyOptions, verify_checksums_in_compaction),
OptionType::kBoolean, OptionVerificationType::kNormal, true,
offsetof(struct MutableCFOptions, verify_checksums_in_compaction)}},
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, 0}},
{"soft_pending_compaction_bytes_limit",
{offsetof(struct ColumnFamilyOptions, soft_pending_compaction_bytes_limit),
OptionType::kUInt64T, OptionVerificationType::kNormal, true,
@ -501,9 +499,7 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
{"memtable_prefix_bloom_probes",
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, 0}},
{"min_partial_merge_operands",
{offsetof(struct ColumnFamilyOptions, min_partial_merge_operands),
OptionType::kUInt32T, OptionVerificationType::kNormal, true,
offsetof(struct MutableCFOptions, min_partial_merge_operands)}},
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, 0}},
{"max_bytes_for_level_base",
{offsetof(struct ColumnFamilyOptions, max_bytes_for_level_base),
OptionType::kUInt64T, OptionVerificationType::kNormal, true,

@ -411,7 +411,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"memtable_factory=SkipListFactory;"
"compression=kNoCompression;"
"bottommost_compression=kDisableCompressionOption;"
"min_partial_merge_operands=7576;"
"level0_stop_writes_trigger=33;"
"num_levels=99;"
"level0_slowdown_writes_trigger=22;"
@ -420,7 +419,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
"soft_rate_limit=530.615385;"
"soft_pending_compaction_bytes_limit=0;"
"max_write_buffer_number_to_maintain=84;"
"verify_checksums_in_compaction=false;"
"merge_operator=aabcxehazrMergeOperator;"
"memtable_prefix_bloom_size_ratio=0.4642;"
"memtable_insert_with_hint_prefix_extractor=rocksdb.CappedPrefix.13;"

@ -174,7 +174,6 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.arena_block_size, 22U);
ASSERT_EQ(new_cf_opt.disable_auto_compactions, true);
ASSERT_EQ(new_cf_opt.compaction_style, kCompactionStyleLevel);
ASSERT_EQ(new_cf_opt.verify_checksums_in_compaction, false);
ASSERT_EQ(new_cf_opt.compaction_options_fifo.max_table_files_size,
static_cast<uint64_t>(23));
ASSERT_EQ(new_cf_opt.max_sequential_skip_in_iterations,
@ -185,7 +184,6 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_cf_opt.memtable_huge_page_size, 28U);
ASSERT_EQ(new_cf_opt.bloom_locality, 29U);
ASSERT_EQ(new_cf_opt.max_successive_merges, 30U);
ASSERT_EQ(new_cf_opt.min_partial_merge_operands, 31U);
ASSERT_TRUE(new_cf_opt.prefix_extractor != nullptr);
ASSERT_EQ(new_cf_opt.optimize_filters_for_hits, true);
ASSERT_EQ(std::string(new_cf_opt.prefix_extractor->Name()),

@ -303,7 +303,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
cf_opt->optimize_filters_for_hits = rnd->Uniform(2);
cf_opt->paranoid_file_checks = rnd->Uniform(2);
cf_opt->purge_redundant_kvs_while_flush = rnd->Uniform(2);
cf_opt->verify_checksums_in_compaction = rnd->Uniform(2);
cf_opt->force_consistency_checks = rnd->Uniform(2);
// double options
@ -339,7 +338,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
// uint32_t options
cf_opt->bloom_locality = rnd->Uniform(10000);
cf_opt->min_partial_merge_operands = rnd->Uniform(10000);
cf_opt->max_bytes_for_level_base = rnd->Uniform(10000);
// uint64_t options

Loading…
Cancel
Save