NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change some RocksDB default options

Browse Source 
Summary: Change some RocksDB default options to make it more friendly to server workloads.

Test Plan: Run all existing tests

Reviewers: yhchiang, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: sumeet, muthu, benj, MarkCallaghan, igor, leveldb, andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D55941
main
sdong 9 years ago
parent a558830f8f
commit 2feafa3db9
12 changed files with 164 additions and 138 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      DEFAULT_OPTIONS_HISTORY.md
  2. 4
      HISTORY.md
  3. 20
      db/compaction_picker_test.cc
  4. 15
      db/db_compaction_filter_test.cc
  5. 38
      db/db_compaction_test.cc
  6. 18
      db/db_inplace_update_test.cc
  7. 59
      db/db_test.cc
  8. 2
      db/db_test_util.cc
  9. 44
      db/db_universal_compaction_test.cc
  10. 32
      include/rocksdb/options.h
  11. 44
      util/options.cc
  12. 20
      util/options_test.cc

6
DEFAULT_OPTIONS_HISTORY.md
Unescape View File

@ -0,0 +1,6 @@
## Unreleased
* options.write_buffer_size changes from 4MB to 64MB
* options.target_file_size_base changes from 2MB to 64MB
* options.max_bytes_for_level_base changes from 10MB to 256MB
* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB
* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB

4
HISTORY.md
Unescape View File

@ -1,4 +1,8 @@
# Rocksdb Change Log # Rocksdb Change Log
## Unreleased
### Public API Change
* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
## 4.6.0 (3/10/2016) ## 4.6.0 (3/10/2016)
### Public API Changes ### Public API Changes
* Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier. * Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.

20
db/compaction_picker_test.cc
Unescape View File

@ -195,6 +195,7 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
NewVersionStorage(6, kCompactionStyleLevel); NewVersionStorage(6, kCompactionStyleLevel);
mutable_cf_options_.target_file_size_base = 10000000; mutable_cf_options_.target_file_size_base = 10000000;
mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.target_file_size_multiplier = 10;
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
Add(0, 1U, "150", "200", 1000000000U); Add(0, 1U, "150", "200", 1000000000U);
// Level 1 score 1.2 // Level 1 score 1.2
Add(1, 66U, "150", "200", 6000000U); Add(1, 66U, "150", "200", 6000000U);
@ -491,6 +492,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
NewVersionStorage(6, kCompactionStyleLevel); NewVersionStorage(6, kCompactionStyleLevel);
mutable_cf_options_.target_file_size_base = 10000000; mutable_cf_options_.target_file_size_base = 10000000;
mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.target_file_size_multiplier = 10;
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
mutable_cf_options_.compaction_pri = kMinOverlappingRatio; mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
Add(2, 6U, "150", "179", 50000000U); Add(2, 6U, "150", "179", 50000000U);
@ -517,6 +519,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
NewVersionStorage(6, kCompactionStyleLevel); NewVersionStorage(6, kCompactionStyleLevel);
mutable_cf_options_.target_file_size_base = 10000000; mutable_cf_options_.target_file_size_base = 10000000;
mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.target_file_size_multiplier = 10;
mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
mutable_cf_options_.compaction_pri = kMinOverlappingRatio; mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
Add(2, 6U, "150", "175", Add(2, 6U, "150", "175",
@ -544,20 +547,21 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) { TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
NewVersionStorage(6, kCompactionStyleLevel); NewVersionStorage(6, kCompactionStyleLevel);
mutable_cf_options_.target_file_size_base = 10000000; mutable_cf_options_.max_bytes_for_level_base = 10000000;
mutable_cf_options_.target_file_size_multiplier = 10; mutable_cf_options_.max_bytes_for_level_multiplier = 10;
mutable_cf_options_.compaction_pri = kMinOverlappingRatio; mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
// file 7 and 8 over lap with the same file, but file 8 is smaller so // file 7 and 8 over lap with the same file, but file 8 is smaller so
// it will be picked. // it will be picked.
Add(2, 6U, "150", "175", 60000000U); // Overlaps with file 26, 27 Add(2, 6U, "150", "167", 60000000U); // Overlaps with file 26, 27
Add(2, 7U, "176", "200", 60000000U); // Overlaps with file 27 Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27
Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 27 Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28, but the file
// itself is larger. Should be picked.
Add(3, 26U, "160", "165", 260000000U); Add(3, 26U, "160", "165", 260000000U);
Add(3, 26U, "166", "170", 260000000U); Add(3, 27U, "166", "170", 260000000U);
Add(3, 27U, "180", "400", 260000000U); Add(3, 28U, "180", "400", 260000000U);
Add(3, 28U, "401", "500", 260000000U); Add(3, 29U, "401", "500", 260000000U);
UpdateVersionStorageInfo(); UpdateVersionStorageInfo();
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction( std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(

15
db/db_compaction_filter_test.cc
Unescape View File

@ -368,11 +368,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
// entries are deleted. The compaction should create bunch of 'DeleteFile' // entries are deleted. The compaction should create bunch of 'DeleteFile'
// entries in VersionEdit, but none of the 'AddFile's. // entries in VersionEdit, but none of the 'AddFile's.
TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) { TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
Options options; Options options = CurrentOptions();
options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>(); options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.create_if_missing = true; options.create_if_missing = true;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
// put some data // put some data
@ -400,11 +399,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) { TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) {
do { do {
Options options; Options options = CurrentOptions();
options.num_levels = 3; options.num_levels = 3;
options.compaction_filter_factory = options.compaction_filter_factory =
std::make_shared<ChangeFilterFactory>(); std::make_shared<ChangeFilterFactory>();
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// Write 100K+1 keys, these are written to a few files // Write 100K+1 keys, these are written to a few files
@ -467,8 +465,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) {
PutFixed64(&three, 3); PutFixed64(&three, 3);
PutFixed64(&four, 4); PutFixed64(&four, 4);
Options options; Options options = CurrentOptions();
options = CurrentOptions(options);
options.create_if_missing = true; options.create_if_missing = true;
options.merge_operator = MergeOperators::CreateUInt64AddOperator(); options.merge_operator = MergeOperators::CreateUInt64AddOperator();
options.num_levels = 3; options.num_levels = 3;
@ -621,11 +618,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextCfId) {
// Compaction filters should only be applied to records that are newer than the // Compaction filters should only be applied to records that are newer than the
// latest snapshot. This test inserts records and applies a delete filter. // latest snapshot. This test inserts records and applies a delete filter.
TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) { TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
Options options; Options options = CurrentOptions();
options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>(); options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.create_if_missing = true; options.create_if_missing = true;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
// Put some data. // Put some data.
@ -659,11 +655,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
// records newer than the snapshot will also be processed // records newer than the snapshot will also be processed
TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) { TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
std::string five = ToString(5); std::string five = ToString(5);
Options options; Options options = CurrentOptions();
options.compaction_filter_factory = std::make_shared<DeleteISFilterFactory>(); options.compaction_filter_factory = std::make_shared<DeleteISFilterFactory>();
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.create_if_missing = true; options.create_if_missing = true;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
// Put some data. // Put some data.

38
db/db_compaction_test.cc
Unescape View File

@ -288,8 +288,7 @@ TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
} }
TEST_F(DBCompactionTest, TestTableReaderForCompaction) { TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
Options options; Options options = CurrentOptions();
options = CurrentOptions(options);
options.env = env_; options.env = env_;
options.new_table_reader_for_compaction_inputs = true; options.new_table_reader_for_compaction_inputs = true;
options.max_open_files = 100; options.max_open_files = 100;
@ -493,14 +492,13 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
TEST_P(DBCompactionTestWithParam, CompactionTrigger) { TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
const int kNumKeysPerFile = 100; const int kNumKeysPerFile = 100;
Options options; Options options = CurrentOptions();
options.write_buffer_size = 110 << 10; // 110KB options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10; options.arena_block_size = 4 << 10;
options.num_levels = 3; options.num_levels = 3;
options.level0_file_num_compaction_trigger = 3; options.level0_file_num_compaction_trigger = 3;
options.max_subcompactions = max_subcompactions_; options.max_subcompactions = max_subcompactions_;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
Random rnd(301); Random rnd(301);
@ -538,7 +536,7 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
// and see number of compactions scheduled to be less than allowed. // and see number of compactions scheduled to be less than allowed.
const int kNumKeysPerFile = 100; const int kNumKeysPerFile = 100;
Options options; Options options = CurrentOptions();
options.write_buffer_size = 110 << 10; // 110KB options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10; options.arena_block_size = 4 << 10;
options.num_levels = 3; options.num_levels = 3;
@ -549,7 +547,6 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
options.base_background_compactions = 1; options.base_background_compactions = 1;
options.max_background_compactions = 3; options.max_background_compactions = 3;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile)); options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options = CurrentOptions(options);
// Block all threads in thread pool. // Block all threads in thread pool.
const size_t kTotalTasks = 4; const size_t kTotalTasks = 4;
@ -632,10 +629,9 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
} }
TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) { TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
Options options; Options options = CurrentOptions();
options.write_buffer_size = 100000000; // Large write buffer options.write_buffer_size = 100000000; // Large write buffer
options.max_subcompactions = max_subcompactions_; options.max_subcompactions = max_subcompactions_;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
Random rnd(301); Random rnd(301);
@ -662,9 +658,8 @@ TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
TEST_F(DBCompactionTest, MinorCompactionsHappen) { TEST_F(DBCompactionTest, MinorCompactionsHappen) {
do { do {
Options options; Options options = CurrentOptions();
options.write_buffer_size = 10000; options.write_buffer_size = 10000;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
const int N = 500; const int N = 500;
@ -689,7 +684,7 @@ TEST_F(DBCompactionTest, MinorCompactionsHappen) {
} }
TEST_F(DBCompactionTest, ZeroSeqIdCompaction) { TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleLevel; options.compaction_style = kCompactionStyleLevel;
options.level0_file_num_compaction_trigger = 3; options.level0_file_num_compaction_trigger = 3;
@ -703,7 +698,6 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
const size_t key_len = const size_t key_len =
static_cast<size_t>(compact_opt.output_file_size_limit) / 5; static_cast<size_t>(compact_opt.output_file_size_limit) / 5;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
std::vector<const Snapshot*> snaps; std::vector<const Snapshot*> snaps;
@ -747,10 +741,8 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
// if the database is shutdown during the memtable compaction. // if the database is shutdown during the memtable compaction.
TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) { TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
do { do {
Options options; Options options = CurrentOptions();
options.env = env_; options.env = env_;
options.write_buffer_size = 1000000;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// Trigger a long memtable compaction and reopen the database during it // Trigger a long memtable compaction and reopen the database during it
@ -774,10 +766,9 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
[&](void* arg) { trivial_move++; }); [&](void* arg) { trivial_move++; });
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); rocksdb::SyncPoint::GetInstance()->EnableProcessing();
Options options; Options options = CurrentOptions();
options.write_buffer_size = 100000000; options.write_buffer_size = 100000000;
options.max_subcompactions = max_subcompactions_; options.max_subcompactions = max_subcompactions_;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
int32_t num_keys = 80; int32_t num_keys = 80;
@ -1369,10 +1360,9 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
[&](void* arg) { non_trivial_move++; }); [&](void* arg) { non_trivial_move++; });
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); rocksdb::SyncPoint::GetInstance()->EnableProcessing();
Options options; Options options = CurrentOptions();
options.write_buffer_size = 100000000; options.write_buffer_size = 100000000;
options.max_subcompactions = max_subcompactions_; options.max_subcompactions = max_subcompactions_;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
int32_t value_size = 10 * 1024; // 10 KB int32_t value_size = 10 * 1024; // 10 KB
@ -1661,7 +1651,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
int max_key_universal_insert = 600; int max_key_universal_insert = 600;
// Stage 1: generate a db with level compaction // Stage 1: generate a db with level compaction
Options options; Options options = CurrentOptions();
options.write_buffer_size = 110 << 10; // 110KB options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10; options.arena_block_size = 4 << 10;
options.num_levels = 4; options.num_levels = 4;
@ -1671,7 +1661,6 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
options.target_file_size_base = 200 << 10; // 200KB options.target_file_size_base = 200 << 10; // 200KB
options.target_file_size_multiplier = 1; options.target_file_size_multiplier = 1;
options.max_subcompactions = max_subcompactions_; options.max_subcompactions = max_subcompactions_;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
for (int i = 0; i <= max_key_level_insert; i++) { for (int i = 0; i <= max_key_level_insert; i++) {
@ -2386,10 +2375,9 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
[&](void* arg) { non_trivial_move++; }); [&](void* arg) { non_trivial_move++; });
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); rocksdb::SyncPoint::GetInstance()->EnableProcessing();
Options options; Options options = CurrentOptions();
options.write_buffer_size = 100000000; options.write_buffer_size = 100000000;
options.max_subcompactions = max_subcompactions_; options.max_subcompactions = max_subcompactions_;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
int32_t value_size = 10 * 1024; // 10 KB int32_t value_size = 10 * 1024; // 10 KB
@ -2479,14 +2467,14 @@ class CompactionPriTest : public DBTestBase,
}; };
TEST_P(CompactionPriTest, Test) { TEST_P(CompactionPriTest, Test) {
Options options; Options options = CurrentOptions();
options.write_buffer_size = 16 * 1024; options.write_buffer_size = 16 * 1024;
options.compaction_pri = static_cast<CompactionPri>(compaction_pri_); options.compaction_pri = static_cast<CompactionPri>(compaction_pri_);
options.hard_pending_compaction_bytes_limit = 256 * 1024; options.hard_pending_compaction_bytes_limit = 256 * 1024;
options.max_bytes_for_level_base = 64 * 1024; options.max_bytes_for_level_base = 64 * 1024;
options.max_bytes_for_level_multiplier = 4; options.max_bytes_for_level_multiplier = 4;
options.compression = kNoCompression; options.compression = kNoCompression;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
Random rnd(301); Random rnd(301);

18
db/db_inplace_update_test.cc
Unescape View File

@ -18,12 +18,11 @@ class DBTestInPlaceUpdate : public DBTestBase {
TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) { TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
do { do {
Options options; Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
options.inplace_update_support = true; options.inplace_update_support = true;
options.env = env_; options.env = env_;
options.write_buffer_size = 100000; options.write_buffer_size = 100000;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of smaller size // Update key with values of smaller size
@ -41,12 +40,11 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) { TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
do { do {
Options options; Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
options.inplace_update_support = true; options.inplace_update_support = true;
options.env = env_; options.env = env_;
options.write_buffer_size = 100000; options.write_buffer_size = 100000;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of larger size // Update key with values of larger size
@ -64,7 +62,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) { TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
do { do {
Options options; Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
options.inplace_update_support = true; options.inplace_update_support = true;
@ -72,7 +70,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
options.write_buffer_size = 100000; options.write_buffer_size = 100000;
options.inplace_callback = options.inplace_callback =
rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerSize; rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerSize;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of smaller size // Update key with values of smaller size
@ -92,7 +89,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) { TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
do { do {
Options options; Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
options.inplace_update_support = true; options.inplace_update_support = true;
@ -100,7 +97,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
options.write_buffer_size = 100000; options.write_buffer_size = 100000;
options.inplace_callback = options.inplace_callback =
rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerVarintSize; rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerVarintSize;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of smaller varint size // Update key with values of smaller varint size
@ -120,7 +116,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) { TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
do { do {
Options options; Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
options.inplace_update_support = true; options.inplace_update_support = true;
@ -128,7 +124,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
options.write_buffer_size = 100000; options.write_buffer_size = 100000;
options.inplace_callback = options.inplace_callback =
rocksdb::DBTestInPlaceUpdate::updateInPlaceLargerSize; rocksdb::DBTestInPlaceUpdate::updateInPlaceLargerSize;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// Update key with values of larger size // Update key with values of larger size
@ -146,7 +141,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) { TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
do { do {
Options options; Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
options.inplace_update_support = true; options.inplace_update_support = true;
@ -154,7 +149,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
options.write_buffer_size = 100000; options.write_buffer_size = 100000;
options.inplace_callback = options.inplace_callback =
rocksdb::DBTestInPlaceUpdate::updateInPlaceNoAction; rocksdb::DBTestInPlaceUpdate::updateInPlaceNoAction;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// Callback function requests no actions from db // Callback function requests no actions from db

59
db/db_test.cc
Unescape View File

@ -228,10 +228,9 @@ TEST_F(DBTest, MemEnvTest) {
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
TEST_F(DBTest, WriteEmptyBatch) { TEST_F(DBTest, WriteEmptyBatch) {
Options options; Options options = CurrentOptions();
options.env = env_; options.env = env_;
options.write_buffer_size = 100000; options.write_buffer_size = 100000;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
ASSERT_OK(Put(1, "foo", "bar")); ASSERT_OK(Put(1, "foo", "bar"));
@ -281,13 +280,12 @@ TEST_F(DBTest, ReadOnlyDB) {
TEST_F(DBTest, CompactedDB) { TEST_F(DBTest, CompactedDB) {
const uint64_t kFileSize = 1 << 20; const uint64_t kFileSize = 1 << 20;
Options options; Options options = CurrentOptions();
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.write_buffer_size = kFileSize; options.write_buffer_size = kFileSize;
options.target_file_size_base = kFileSize; options.target_file_size_base = kFileSize;
options.max_bytes_for_level_base = 1 << 30; options.max_bytes_for_level_base = 1 << 30;
options.compression = kNoCompression; options.compression = kNoCompression;
options = CurrentOptions(options);
Reopen(options); Reopen(options);
// 1 L0 file, use CompactedDB if max_open_files = -1 // 1 L0 file, use CompactedDB if max_open_files = -1
ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1'))); ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1')));
@ -764,11 +762,10 @@ TEST_F(DBTest, DISABLED_VeryLargeValue) {
std::string key1(kKeySize, 'c'); std::string key1(kKeySize, 'c');
std::string key2(kKeySize, 'd'); std::string key2(kKeySize, 'd');
Options options; Options options = CurrentOptions();
options.env = env_; options.env = env_;
options.write_buffer_size = 100000; // Small write buffer options.write_buffer_size = 100000; // Small write buffer
options.paranoid_checks = true; options.paranoid_checks = true;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
ASSERT_OK(Put("boo", "v1")); ASSERT_OK(Put("boo", "v1"));
@ -811,10 +808,8 @@ TEST_F(DBTest, DISABLED_VeryLargeValue) {
TEST_F(DBTest, GetFromImmutableLayer) { TEST_F(DBTest, GetFromImmutableLayer) {
do { do {
Options options; Options options = CurrentOptions();
options.env = env_; options.env = env_;
options.write_buffer_size = 100000; // Small write buffer
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
ASSERT_OK(Put(1, "foo", "v1")); ASSERT_OK(Put(1, "foo", "v1"));
@ -1916,11 +1911,9 @@ TEST_F(DBTest, Recover) {
TEST_F(DBTest, RecoverWithTableHandle) { TEST_F(DBTest, RecoverWithTableHandle) {
do { do {
Options options; Options options = CurrentOptions();
options.create_if_missing = true; options.create_if_missing = true;
options.write_buffer_size = 100;
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -2368,10 +2361,9 @@ TEST_F(DBTest, CompressedCache) {
// Iteration 4: both block cache and compressed cache, but DB is not // Iteration 4: both block cache and compressed cache, but DB is not
// compressed // compressed
for (int iter = 0; iter < 4; iter++) { for (int iter = 0; iter < 4; iter++) {
Options options; Options options = CurrentOptions();
options.write_buffer_size = 64*1024; // small write buffer options.write_buffer_size = 64*1024; // small write buffer
options.statistics = rocksdb::CreateDBStatistics(); options.statistics = rocksdb::CreateDBStatistics();
options = CurrentOptions(options);
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
switch (iter) { switch (iter) {
@ -2636,10 +2628,9 @@ TEST_F(DBTest, MinLevelToCompress2) {
TEST_F(DBTest, RepeatedWritesToSameKey) { TEST_F(DBTest, RepeatedWritesToSameKey) {
do { do {
Options options; Options options = CurrentOptions();
options.env = env_; options.env = env_;
options.write_buffer_size = 100000; // Small write buffer options.write_buffer_size = 100000; // Small write buffer
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// We must have at most one file per level except for level-0, // We must have at most one file per level except for level-0,
@ -2716,11 +2707,10 @@ static bool Between(uint64_t val, uint64_t low, uint64_t high) {
} }
TEST_F(DBTest, ApproximateSizesMemTable) { TEST_F(DBTest, ApproximateSizesMemTable) {
Options options; Options options = CurrentOptions();
options.write_buffer_size = 100000000; // Large write buffer options.write_buffer_size = 100000000; // Large write buffer
options.compression = kNoCompression; options.compression = kNoCompression;
options.create_if_missing = true; options.create_if_missing = true;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
const int N = 128; const int N = 128;
@ -2823,11 +2813,10 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
TEST_F(DBTest, ApproximateSizes) { TEST_F(DBTest, ApproximateSizes) {
do { do {
Options options; Options options = CurrentOptions();
options.write_buffer_size = 100000000; // Large write buffer options.write_buffer_size = 100000000; // Large write buffer
options.compression = kNoCompression; options.compression = kNoCompression;
options.create_if_missing = true; options.create_if_missing = true;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -7843,8 +7832,7 @@ TEST_F(DBTest, MergeTestTime) {
this->env_->addon_time_.store(0); this->env_->addon_time_.store(0);
this->env_->time_elapse_only_sleep_ = true; this->env_->time_elapse_only_sleep_ = true;
this->env_->no_sleep_ = true; this->env_->no_sleep_ = true;
Options options; Options options = CurrentOptions();
options = CurrentOptions(options);
options.statistics = rocksdb::CreateDBStatistics(); options.statistics = rocksdb::CreateDBStatistics();
options.merge_operator.reset(new DelayedMergeOperator(this)); options.merge_operator.reset(new DelayedMergeOperator(this));
DestroyAndReopen(options); DestroyAndReopen(options);
@ -7884,8 +7872,7 @@ TEST_F(DBTest, MergeTestTime) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
TEST_P(DBTestWithParam, MergeCompactionTimeTest) { TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
SetPerfLevel(kEnableTime); SetPerfLevel(kEnableTime);
Options options; Options options = CurrentOptions();
options = CurrentOptions(options);
options.compaction_filter_factory = std::make_shared<KeepFilterFactory>(); options.compaction_filter_factory = std::make_shared<KeepFilterFactory>();
options.statistics = rocksdb::CreateDBStatistics(); options.statistics = rocksdb::CreateDBStatistics();
options.merge_operator.reset(new DelayedMergeOperator(this)); options.merge_operator.reset(new DelayedMergeOperator(this));
@ -7904,14 +7891,13 @@ TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
} }
TEST_P(DBTestWithParam, FilterCompactionTimeTest) { TEST_P(DBTestWithParam, FilterCompactionTimeTest) {
Options options; Options options = CurrentOptions();
options.compaction_filter_factory = options.compaction_filter_factory =
std::make_shared<DelayFilterFactory>(this); std::make_shared<DelayFilterFactory>(this);
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.create_if_missing = true; options.create_if_missing = true;
options.statistics = rocksdb::CreateDBStatistics(); options.statistics = rocksdb::CreateDBStatistics();
options.max_subcompactions = max_subcompactions_; options.max_subcompactions = max_subcompactions_;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
// put some data // put some data
@ -7953,9 +7939,8 @@ TEST_F(DBTest, TestLogCleanup) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
TEST_F(DBTest, EmptyCompactedDB) { TEST_F(DBTest, EmptyCompactedDB) {
Options options; Options options = CurrentOptions();
options.max_open_files = -1; options.max_open_files = -1;
options = CurrentOptions(options);
Close(); Close();
ASSERT_OK(ReadOnlyReopen(options)); ASSERT_OK(ReadOnlyReopen(options));
Status s = Put("new", "value"); Status s = Put("new", "value");
@ -8191,9 +8176,8 @@ TEST_F(DBTest, AutomaticConflictsWithManualCompaction) {
// Github issue #595 // Github issue #595
// Large write batch with column families // Large write batch with column families
TEST_F(DBTest, LargeBatchWithColumnFamilies) { TEST_F(DBTest, LargeBatchWithColumnFamilies) {
Options options; Options options = CurrentOptions();
options.env = env_; options.env = env_;
options = CurrentOptions(options);
options.write_buffer_size = 100000; // Small write buffer options.write_buffer_size = 100000; // Small write buffer
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
int64_t j = 0; int64_t j = 0;
@ -8287,11 +8271,10 @@ TEST_F(DBTest, DelayedWriteRate) {
const int kEntriesPerMemTable = 100; const int kEntriesPerMemTable = 100;
const int kTotalFlushes = 20; const int kTotalFlushes = 20;
Options options; Options options = CurrentOptions();
env_->SetBackgroundThreads(1, Env::LOW); env_->SetBackgroundThreads(1, Env::LOW);
options.env = env_; options.env = env_;
env_->no_sleep_ = true; env_->no_sleep_ = true;
options = CurrentOptions(options);
options.write_buffer_size = 100000000; options.write_buffer_size = 100000000;
options.max_write_buffer_number = 256; options.max_write_buffer_number = 256;
options.max_background_compactions = 1; options.max_background_compactions = 1;
@ -8352,10 +8335,9 @@ TEST_F(DBTest, DelayedWriteRate) {
} }
TEST_F(DBTest, HardLimit) { TEST_F(DBTest, HardLimit) {
Options options; Options options = CurrentOptions();
options.env = env_; options.env = env_;
env_->SetBackgroundThreads(1, Env::LOW); env_->SetBackgroundThreads(1, Env::LOW);
options = CurrentOptions(options);
options.max_write_buffer_number = 256; options.max_write_buffer_number = 256;
options.write_buffer_size = 110 << 10; // 110KB options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 * 1024; options.arena_block_size = 4 * 1024;
@ -8403,9 +8385,8 @@ TEST_F(DBTest, HardLimit) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
TEST_F(DBTest, SoftLimit) { TEST_F(DBTest, SoftLimit) {
Options options; Options options = CurrentOptions();
options.env = env_; options.env = env_;
options = CurrentOptions(options);
options.write_buffer_size = 100000; // Small write buffer options.write_buffer_size = 100000; // Small write buffer
options.max_write_buffer_number = 256; options.max_write_buffer_number = 256;
options.level0_file_num_compaction_trigger = 1; options.level0_file_num_compaction_trigger = 1;
@ -8521,9 +8502,8 @@ TEST_F(DBTest, SoftLimit) {
} }
TEST_F(DBTest, LastWriteBufferDelay) { TEST_F(DBTest, LastWriteBufferDelay) {
Options options; Options options = CurrentOptions();
options.env = env_; options.env = env_;
options = CurrentOptions(options);
options.write_buffer_size = 100000; options.write_buffer_size = 100000;
options.max_write_buffer_number = 4; options.max_write_buffer_number = 4;
options.delayed_write_rate = 20000; options.delayed_write_rate = 20000;
@ -10108,9 +10088,8 @@ INSTANTIATE_TEST_CASE_P(DBTestWithParam, DBTestWithParam,
::testing::Bool())); ::testing::Bool()));
TEST_F(DBTest, PauseBackgroundWorkTest) { TEST_F(DBTest, PauseBackgroundWorkTest) {
Options options; Options options = CurrentOptions();
options.write_buffer_size = 100000; // Small write buffer options.write_buffer_size = 100000; // Small write buffer
options = CurrentOptions(options);
Reopen(options); Reopen(options);
std::vector<std::thread> threads; std::vector<std::thread> threads;

2
db/db_test_util.cc
Unescape View File

@ -210,6 +210,8 @@ Options DBTestBase::CurrentOptions(
const anon::OptionsOverride& options_override) { const anon::OptionsOverride& options_override) {
Options options; Options options;
options.write_buffer_size = 4090 * 4096; options.write_buffer_size = 4090 * 4096;
options.target_file_size_base = 2 * 1024 * 1024;
options.max_bytes_for_level_base = 10 * 1024 * 1024;
return CurrentOptions(options, options_override); return CurrentOptions(options, options_override);
} }

44
db/db_universal_compaction_test.cc
Unescape View File

@ -124,8 +124,7 @@ class DelayFilterFactory : public CompactionFilterFactory {
// Make sure we don't trigger a problem if the trigger conditon is given // Make sure we don't trigger a problem if the trigger conditon is given
// to be 0, which is invalid. // to be 0, which is invalid.
TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) { TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
Options options; Options options = CurrentOptions();
options = CurrentOptions(options);
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.num_levels = num_levels_; options.num_levels = num_levels_;
@ -160,8 +159,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
} }
TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) { TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
Options options; Options options = CurrentOptions();
options = CurrentOptions(options);
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.size_ratio = 5; options.compaction_options_universal.size_ratio = 5;
options.num_levels = num_levels_; options.num_levels = num_levels_;
@ -337,13 +335,12 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
} }
TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) { TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.num_levels = num_levels_; options.num_levels = num_levels_;
options.write_buffer_size = 100 << 10; // 100KB options.write_buffer_size = 100 << 10; // 100KB
options.target_file_size_base = 32 << 10; // 32KB options.target_file_size_base = 32 << 10; // 32KB
options.level0_file_num_compaction_trigger = 3; options.level0_file_num_compaction_trigger = 3;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -388,12 +385,12 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
ChangeCompactOptions(); ChangeCompactOptions();
Options options; Options options;
options.create_if_missing = true; options.create_if_missing = true;
options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
options.compaction_style = kCompactionStyleLevel; options.compaction_style = kCompactionStyleLevel;
options.num_levels = 1; options.num_levels = 1;
options.target_file_size_base = options.write_buffer_size; options.target_file_size_base = options.write_buffer_size;
options.compression = kNoCompression; options.compression = kNoCompression;
options = CurrentOptions(options); options = CurrentOptions(options);
options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal); ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
Random rnd(301); Random rnd(301);
@ -451,12 +448,11 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
} }
TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) { TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 100 << 10; // 100KB options.write_buffer_size = 100 << 10; // 100KB
options.num_levels = 7; options.num_levels = 7;
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
// Generate 3 overlapping files // Generate 3 overlapping files
@ -496,14 +492,13 @@ class DBTestUniversalCompactionMultiLevels
}; };
TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) { TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.num_levels = num_levels_; options.num_levels = num_levels_;
options.write_buffer_size = 100 << 10; // 100KB options.write_buffer_size = 100 << 10; // 100KB
options.level0_file_num_compaction_trigger = 8; options.level0_file_num_compaction_trigger = 8;
options.max_background_compactions = 3; options.max_background_compactions = 3;
options.target_file_size_base = 32 * 1024; options.target_file_size_base = 32 * 1024;
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
// Trigger compaction if size amplification exceeds 110% // Trigger compaction if size amplification exceeds 110%
@ -539,7 +534,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
}); });
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); rocksdb::SyncPoint::GetInstance()->EnableProcessing();
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.allow_trivial_move = true; options.compaction_options_universal.allow_trivial_move = true;
options.num_levels = 3; options.num_levels = 3;
@ -547,7 +542,6 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
options.level0_file_num_compaction_trigger = 3; options.level0_file_num_compaction_trigger = 3;
options.max_background_compactions = 2; options.max_background_compactions = 2;
options.target_file_size_base = 32 * 1024; options.target_file_size_base = 32 * 1024;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -586,7 +580,7 @@ class DBTestUniversalCompactionParallel :
}; };
TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) { TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.num_levels = num_levels_; options.num_levels = num_levels_;
options.write_buffer_size = 1 << 10; // 1KB options.write_buffer_size = 1 << 10; // 1KB
@ -595,7 +589,6 @@ TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
options.max_background_flushes = 3; options.max_background_flushes = 3;
options.target_file_size_base = 1 * 1024; options.target_file_size_base = 1 * 1024;
options.compaction_options_universal.max_size_amplification_percent = 110; options.compaction_options_universal.max_size_amplification_percent = 110;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -652,7 +645,7 @@ INSTANTIATE_TEST_CASE_P(DBTestUniversalCompactionParallel,
::testing::Bool())); ::testing::Bool()));
TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) { TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 105 << 10; // 105KB options.write_buffer_size = 105 << 10; // 105KB
options.arena_block_size = 4 << 10; // 4KB options.arena_block_size = 4 << 10; // 4KB
@ -660,7 +653,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
options.level0_file_num_compaction_trigger = 4; options.level0_file_num_compaction_trigger = 4;
options.num_levels = num_levels_; options.num_levels = num_levels_;
options.compaction_options_universal.compression_size_percent = -1; options.compaction_options_universal.compression_size_percent = -1;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -773,14 +765,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
return; return;
} }
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 100 << 10; // 100KB options.write_buffer_size = 100 << 10; // 100KB
options.target_file_size_base = 32 << 10; // 32KB options.target_file_size_base = 32 << 10; // 32KB
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
options.num_levels = num_levels_; options.num_levels = num_levels_;
options.compaction_options_universal.compression_size_percent = 70; options.compaction_options_universal.compression_size_percent = 70;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
Random rnd(301); Random rnd(301);
@ -841,14 +832,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
if (!Snappy_Supported()) { if (!Snappy_Supported()) {
return; return;
} }
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 100 << 10; // 100KB options.write_buffer_size = 100 << 10; // 100KB
options.target_file_size_base = 32 << 10; // 32KB options.target_file_size_base = 32 << 10; // 32KB
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
options.num_levels = num_levels_; options.num_levels = num_levels_;
options.compaction_options_universal.compression_size_percent = 95; options.compaction_options_universal.compression_size_percent = 95;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
Random rnd(301); Random rnd(301);
@ -884,7 +874,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
}); });
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); rocksdb::SyncPoint::GetInstance()->EnableProcessing();
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.allow_trivial_move = true; options.compaction_options_universal.allow_trivial_move = true;
options.num_levels = 2; options.num_levels = 2;
@ -892,7 +882,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
options.level0_file_num_compaction_trigger = 3; options.level0_file_num_compaction_trigger = 3;
options.max_background_compactions = 1; options.max_background_compactions = 1;
options.target_file_size_base = 32 * 1024; options.target_file_size_base = 32 * 1024;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -929,7 +918,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); rocksdb::SyncPoint::GetInstance()->EnableProcessing();
Options options; Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.allow_trivial_move = true; options.compaction_options_universal.allow_trivial_move = true;
options.num_levels = 15; options.num_levels = 15;
@ -937,7 +926,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
options.level0_file_num_compaction_trigger = 8; options.level0_file_num_compaction_trigger = 8;
options.max_background_compactions = 4; options.max_background_compactions = 4;
options.target_file_size_base = 64 * 1024; options.target_file_size_base = 64 * 1024;
options = CurrentOptions(options);
DestroyAndReopen(options); DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
@ -963,7 +951,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
} }
TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) { TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
Options options; Options options = CurrentOptions();
options.db_paths.emplace_back(dbname_, 300 * 1024); options.db_paths.emplace_back(dbname_, 300 * 1024);
options.db_paths.emplace_back(dbname_ + "_2", 300 * 1024); options.db_paths.emplace_back(dbname_ + "_2", 300 * 1024);
options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024); options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
@ -976,7 +964,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
options.arena_block_size = 4 << 10; options.arena_block_size = 4 << 10;
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
options.num_levels = 1; options.num_levels = 1;
options = CurrentOptions(options);
std::vector<std::string> filenames; std::vector<std::string> filenames;
env_->GetChildren(options.db_paths[1].path, &filenames); env_->GetChildren(options.db_paths[1].path, &filenames);
@ -1163,7 +1150,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
if (!Snappy_Supported()) { if (!Snappy_Supported()) {
return; return;
} }
Options options; Options options = CurrentOptions();
options.db_paths.emplace_back(dbname_, 500 * 1024); options.db_paths.emplace_back(dbname_, 500 * 1024);
options.db_paths.emplace_back(dbname_ + "_2", 1024 * 1024 * 1024); options.db_paths.emplace_back(dbname_ + "_2", 1024 * 1024 * 1024);
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
@ -1175,7 +1162,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
options.num_levels = 1; options.num_levels = 1;
options.memtable_factory.reset( options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1)); new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options = CurrentOptions(options);
std::vector<std::string> filenames; std::vector<std::string> filenames;
env_->GetChildren(options.db_paths[1].path, &filenames); env_->GetChildren(options.db_paths[1].path, &filenames);

32
include/rocksdb/options.h
Unescape View File

@ -158,7 +158,15 @@ struct DbPath {
struct Options; struct Options;
struct ColumnFamilyOptions { struct ColumnFamilyOptions {
// The function recovers options to a previous version. Only 4.6 or later
// versions are supported.
ColumnFamilyOptions* OldDefaults(int rocksdb_major_version = 4,
int rocksdb_minor_version = 6);
// Some functions that make it easier to optimize RocksDB // Some functions that make it easier to optimize RocksDB
// Use if if your DB is very small (like under 1GB) and you don't want to
// spend lots of memory for memtables.
ColumnFamilyOptions* OptimizeForSmallDb();
// Use this if you don't need to keep the data sorted, i.e. you'll never use // Use this if you don't need to keep the data sorted, i.e. you'll never use
// an iterator, only Put() and Get() API calls // an iterator, only Put() and Get() API calls
@ -254,7 +262,7 @@ struct ColumnFamilyOptions {
// Note that write_buffer_size is enforced per column family. // Note that write_buffer_size is enforced per column family.
// See db_write_buffer_size for sharing memory across column families. // See db_write_buffer_size for sharing memory across column families.
// //
// Default: 4MB // Default: 64MB
// //
// Dynamically changeable through SetOptions() API // Dynamically changeable through SetOptions() API
size_t write_buffer_size; size_t write_buffer_size;
@ -400,7 +408,7 @@ struct ColumnFamilyOptions {
// be 2MB, and each file on level 2 will be 20MB, // be 2MB, and each file on level 2 will be 20MB,
// and each file on level-3 will be 200MB. // and each file on level-3 will be 200MB.
// //
// Default: 2MB. // Default: 64MB.
// //
// Dynamically changeable through SetOptions() API // Dynamically changeable through SetOptions() API
uint64_t target_file_size_base; uint64_t target_file_size_base;
@ -415,12 +423,12 @@ struct ColumnFamilyOptions {
// max_bytes_for_level_base is the max total for level-1. // max_bytes_for_level_base is the max total for level-1.
// Maximum number of bytes for level L can be calculated as // Maximum number of bytes for level L can be calculated as
// (max_bytes_for_level_base) * (max_bytes_for_level_multiplier ^ (L-1)) // (max_bytes_for_level_base) * (max_bytes_for_level_multiplier ^ (L-1))
// For example, if max_bytes_for_level_base is 20MB, and if // For example, if max_bytes_for_level_base is 200MB, and if
// max_bytes_for_level_multiplier is 10, total data size for level-1 // max_bytes_for_level_multiplier is 10, total data size for level-1
// will be 20MB, total file size for level-2 will be 200MB, // will be 2GB, total file size for level-2 will be 20GB,
// and total file size for level-3 will be 2GB. // and total file size for level-3 will be 200GB.
// //
// Default: 10MB. // Default: 256MB.
// //
// Dynamically changeable through SetOptions() API // Dynamically changeable through SetOptions() API
uint64_t max_bytes_for_level_base; uint64_t max_bytes_for_level_base;
@ -538,13 +546,13 @@ struct ColumnFamilyOptions {
// All writes will be slowed down to at least delayed_write_rate if estimated // All writes will be slowed down to at least delayed_write_rate if estimated
// bytes needed to be compaction exceed this threshold. // bytes needed to be compaction exceed this threshold.
// //
// Default: 0 (disabled) // Default: 64GB
uint64_t soft_pending_compaction_bytes_limit; uint64_t soft_pending_compaction_bytes_limit;
// All writes are stopped if estimated bytes needed to be compaction exceed // All writes are stopped if estimated bytes needed to be compaction exceed
// this threshold. // this threshold.
// //
// Default: 0 (disabled) // Default: 256GB
uint64_t hard_pending_compaction_bytes_limit; uint64_t hard_pending_compaction_bytes_limit;
// DEPRECATED -- this options is no longer used // DEPRECATED -- this options is no longer used
@ -795,6 +803,10 @@ struct ColumnFamilyOptions {
}; };
struct DBOptions { struct DBOptions {
// The function recovers options to the option as in version 4.6.
DBOptions* OldDefaults(int rocksdb_major_version = 4,
int rocksdb_minor_version = 6);
// Some functions that make it easier to optimize RocksDB // Some functions that make it easier to optimize RocksDB
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
@ -1303,6 +1315,10 @@ struct Options : public DBOptions, public ColumnFamilyOptions {
const ColumnFamilyOptions& column_family_options) const ColumnFamilyOptions& column_family_options)
: DBOptions(db_options), ColumnFamilyOptions(column_family_options) {} : DBOptions(db_options), ColumnFamilyOptions(column_family_options) {}
// The function recovers options to the option as in version 4.6.
Options* OldDefaults(int rocksdb_major_version = 4,
int rocksdb_minor_version = 6);
void Dump(Logger* log) const; void Dump(Logger* log) const;
void DumpCFOptions(Logger* log) const; void DumpCFOptions(Logger* log) const;

44
util/options.cc
Unescape View File

@ -83,7 +83,7 @@ ColumnFamilyOptions::ColumnFamilyOptions()
merge_operator(nullptr), merge_operator(nullptr),
compaction_filter(nullptr), compaction_filter(nullptr),
compaction_filter_factory(nullptr), compaction_filter_factory(nullptr),
write_buffer_size(4 << 20), write_buffer_size(64 << 20),
max_write_buffer_number(2), max_write_buffer_number(2),
min_write_buffer_number_to_merge(1), min_write_buffer_number_to_merge(1),
max_write_buffer_number_to_maintain(0), max_write_buffer_number_to_maintain(0),
@ -93,9 +93,9 @@ ColumnFamilyOptions::ColumnFamilyOptions()
level0_file_num_compaction_trigger(4), level0_file_num_compaction_trigger(4),
level0_slowdown_writes_trigger(20), level0_slowdown_writes_trigger(20),
level0_stop_writes_trigger(24), level0_stop_writes_trigger(24),
target_file_size_base(2 * 1048576), target_file_size_base(64 * 1048576),
target_file_size_multiplier(1), target_file_size_multiplier(1),
max_bytes_for_level_base(10 * 1048576), max_bytes_for_level_base(256 * 1048576),
level_compaction_dynamic_level_bytes(false), level_compaction_dynamic_level_bytes(false),
max_bytes_for_level_multiplier(10), max_bytes_for_level_multiplier(10),
max_bytes_for_level_multiplier_additional(num_levels, 1), max_bytes_for_level_multiplier_additional(num_levels, 1),
@ -104,8 +104,8 @@ ColumnFamilyOptions::ColumnFamilyOptions()
max_grandparent_overlap_factor(10), max_grandparent_overlap_factor(10),
soft_rate_limit(0.0), soft_rate_limit(0.0),
hard_rate_limit(0.0), hard_rate_limit(0.0),
soft_pending_compaction_bytes_limit(0), soft_pending_compaction_bytes_limit(64 * 1073741824ul),
hard_pending_compaction_bytes_limit(0), hard_pending_compaction_bytes_limit(256 * 1073741824ul),
rate_limit_delay_max_milliseconds(1000), rate_limit_delay_max_milliseconds(1000),
arena_block_size(0), arena_block_size(0),
disable_auto_compactions(false), disable_auto_compactions(false),
@ -663,8 +663,40 @@ Options::PrepareForBulkLoad()
return this; return this;
} }
#ifndef ROCKSDB_LITE Options* Options::OldDefaults(int rocksdb_major_version,
int rocksdb_minor_version) {
ColumnFamilyOptions::OldDefaults(rocksdb_major_version,
rocksdb_minor_version);
DBOptions::OldDefaults(rocksdb_major_version, rocksdb_minor_version);
return this;
}
DBOptions* DBOptions::OldDefaults(int rocksdb_major_version,
int rocksdb_minor_version) {
return this;
}
ColumnFamilyOptions* ColumnFamilyOptions::OldDefaults(
int rocksdb_major_version, int rocksdb_minor_version) {
write_buffer_size = 4 << 20;
target_file_size_base = 2 * 1048576;
max_bytes_for_level_base = 10 * 1048576;
soft_pending_compaction_bytes_limit = 0;
hard_pending_compaction_bytes_limit = 0;
return this;
}
// Optimization functions // Optimization functions
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForSmallDb() {
write_buffer_size = 2 << 20;
target_file_size_base = 2 * 1048576;
max_bytes_for_level_base = 10 * 1048576;
soft_pending_compaction_bytes_limit = 256 * 1048576;
hard_pending_compaction_bytes_limit = 1073741824ul;
return this;
}
#ifndef ROCKSDB_LITE
ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup( ColumnFamilyOptions* ColumnFamilyOptions::OptimizeForPointLookup(
uint64_t block_cache_size_mb) { uint64_t block_cache_size_mb) {
prefix_extractor.reset(NewNoopTransform()); prefix_extractor.reset(NewNoopTransform());

20
util/options_test.cc
Unescape View File

@ -1270,6 +1270,26 @@ TEST_F(OptionsParserTest, DifferentDefault) {
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_OK(parser.Parse(kOptionsFileName, env_.get())); ASSERT_OK(parser.Parse(kOptionsFileName, env_.get()));
Options old_default_opts;
old_default_opts.OldDefaults();
ASSERT_EQ(10 * 1048576, old_default_opts.max_bytes_for_level_base);
Options old_default_opts46;
old_default_opts46.OldDefaults();
ASSERT_EQ(10 * 1048576, old_default_opts46.max_bytes_for_level_base);
ColumnFamilyOptions old_default_cf_opts;
old_default_cf_opts.OldDefaults();
ASSERT_EQ(2 * 1048576, old_default_cf_opts.target_file_size_base);
ColumnFamilyOptions old_default_cf_opts46;
old_default_cf_opts46.OldDefaults();
ASSERT_EQ(2 * 1048576, old_default_cf_opts46.target_file_size_base);
ColumnFamilyOptions cf_small_opts;
cf_small_opts.OptimizeForSmallDb();
ASSERT_EQ(2 << 20, cf_small_opts.write_buffer_size);
} }
class OptionsSanityCheckTest : public OptionsParserTest { class OptionsSanityCheckTest : public OptionsParserTest {

Write Preview
Loading…
Cancel
Save