From f0804db7f722b0a3bb03f7392d96c5118e6235da Mon Sep 17 00:00:00 2001 From: Sagar Vemuri Date: Thu, 19 Oct 2017 15:19:20 -0700 Subject: [PATCH] Make FIFO compaction options dynamically configurable Summary: ColumnFamilyOptions::compaction_options_fifo and all its sub-fields can be set dynamically now. Some of the ways in which the fifo compaction options can be set are: - `SetOptions({{"compaction_options_fifo", "{max_table_files_size=1024}"}})` - `SetOptions({{"compaction_options_fifo", "{ttl=600;}"}})` - `SetOptions({{"compaction_options_fifo", "{max_table_files_size=1024;ttl=600;}"}})` - `SetOptions({{"compaction_options_fifo", "{max_table_files_size=51;ttl=49;allow_compaction=true;}"}})` Most of the code has been made generic enough so that it could be reused later to make universal options (and other such nested defined-types) dynamic with very few lines of parsing/serializing code changes. Introduced a few new functions like `ParseStruct`, `SerializeStruct` and `GetStringFromStruct`. The duplicate code in `GetStringFromDBOptions` and `GetStringFromColumnFamilyOptions` has been moved into `GetStringFromStruct`. So they become just simple wrappers now. Closes https://github.com/facebook/rocksdb/pull/3006 Differential Revision: D6058619 Pulled By: sagar0 fbshipit-source-id: 1e8f78b3374ca5249bb4f3be8a6d3bb4cbc52f92 --- HISTORY.md | 1 + db/compaction_picker.cc | 44 ++++---- db/compaction_picker_test.cc | 2 +- db/db_options_test.cc | 108 ++++++++++++++++++ db/db_test.cc | 66 +++++++++++ db/version_set.cc | 21 ++-- include/rocksdb/advanced_options.h | 4 + options/cf_options.cc | 1 - options/cf_options.h | 4 +- options/options_helper.cc | 175 ++++++++++++++++++++--------- options/options_helper.h | 22 +++- options/options_parser.cc | 11 ++ options/options_settable_test.cc | 4 +- util/testutil.cc | 4 + 14 files changed, 375 insertions(+), 92 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 6f2c16f80..fbf6fe925 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,6 +6,7 @@ ### New Features * `DBOptions::bytes_per_sync` and `DBOptions::wal_bytes_per_sync` can now be changed dynamically, `DBOptions::wal_bytes_per_sync` will flush all memtables and switch to a new WAL file. * Support dynamic adjustment of rate limit according to demand for background I/O. It can be enabled by passing `true` to the `auto_tuned` parameter in `NewGenericRateLimiter()`. The value passed as `rate_bytes_per_sec` will still be respected as an upper-bound. +* Support dynamically changing `ColumnFamilyOptions::compaction_options_fifo`. ### Bug Fixes * Fix a potential data inconsistency issue during point-in-time recovery. `DB:Open()` will abort if column family inconsistency is found during PIT recovery. diff --git a/db/compaction_picker.cc b/db/compaction_picker.cc index 21571e14f..119a8c453 100644 --- a/db/compaction_picker.cc +++ b/db/compaction_picker.cc @@ -1282,19 +1282,19 @@ uint32_t LevelCompactionBuilder::GetPathId( current_path_size -= level_size; if (cur_level > 0) { if (ioptions.level_compaction_dynamic_level_bytes) { - // Currently, level_compaction_dynamic_level_bytes is ignored when + // Currently, level_compaction_dynamic_level_bytes is ignored when // multiple db paths are specified. https://github.com/facebook/ - // rocksdb/blob/master/db/column_family.cc. - // Still, adding this check to avoid accidentally using + // rocksdb/blob/master/db/column_family.cc. + // Still, adding this check to avoid accidentally using // max_bytes_for_level_multiplier_additional - level_size = static_cast( - level_size * mutable_cf_options.max_bytes_for_level_multiplier); + level_size = static_cast( + level_size * mutable_cf_options.max_bytes_for_level_multiplier); } else { level_size = static_cast( - level_size * mutable_cf_options.max_bytes_for_level_multiplier - * mutable_cf_options.MaxBytesMultiplerAdditional(cur_level)); + level_size * mutable_cf_options.max_bytes_for_level_multiplier * + mutable_cf_options.MaxBytesMultiplerAdditional(cur_level)); } - } + } cur_level++; continue; } @@ -1425,7 +1425,7 @@ uint64_t GetTotalFilesSize( Compaction* FIFOCompactionPicker::PickTTLCompaction( const std::string& cf_name, const MutableCFOptions& mutable_cf_options, VersionStorageInfo* vstorage, LogBuffer* log_buffer) { - assert(ioptions_.compaction_options_fifo.ttl > 0); + assert(mutable_cf_options.compaction_options_fifo.ttl > 0); const int kLevel0 = 0; const std::vector& level_files = vstorage->LevelFiles(kLevel0); @@ -1454,7 +1454,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction( f->fd.table_reader->GetTableProperties()->creation_time; if (creation_time == 0 || creation_time >= - (current_time - ioptions_.compaction_options_fifo.ttl)) { + (current_time - mutable_cf_options.compaction_options_fifo.ttl)) { break; } total_size -= f->compensated_file_size; @@ -1467,7 +1467,8 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction( // 2. there are a few files older than ttl, but deleting them will not bring // the total size to be less than max_table_files_size threshold. if (inputs[0].files.empty() || - total_size > ioptions_.compaction_options_fifo.max_table_files_size) { + total_size > + mutable_cf_options.compaction_options_fifo.max_table_files_size) { return nullptr; } @@ -1493,10 +1494,11 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction( const std::vector& level_files = vstorage->LevelFiles(kLevel0); uint64_t total_size = GetTotalFilesSize(level_files); - if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size || + if (total_size <= + mutable_cf_options.compaction_options_fifo.max_table_files_size || level_files.size() == 0) { // total size not exceeded - if (ioptions_.compaction_options_fifo.allow_compaction && + if (mutable_cf_options.compaction_options_fifo.allow_compaction && level_files.size() > 0) { CompactionInputFiles comp_inputs; if (FindIntraL0Compaction( @@ -1516,11 +1518,12 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction( } } - ROCKS_LOG_BUFFER(log_buffer, - "[%s] FIFO compaction: nothing to do. Total size %" PRIu64 - ", max size %" PRIu64 "\n", - cf_name.c_str(), total_size, - ioptions_.compaction_options_fifo.max_table_files_size); + ROCKS_LOG_BUFFER( + log_buffer, + "[%s] FIFO compaction: nothing to do. Total size %" PRIu64 + ", max size %" PRIu64 "\n", + cf_name.c_str(), total_size, + mutable_cf_options.compaction_options_fifo.max_table_files_size); return nullptr; } @@ -1547,7 +1550,8 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction( "[%s] FIFO compaction: picking file %" PRIu64 " with size %s for deletion", cf_name.c_str(), f->fd.GetNumber(), tmp_fsize); - if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size) { + if (total_size <= + mutable_cf_options.compaction_options_fifo.max_table_files_size) { break; } } @@ -1565,7 +1569,7 @@ Compaction* FIFOCompactionPicker::PickCompaction( assert(vstorage->num_levels() == 1); Compaction* c = nullptr; - if (ioptions_.compaction_options_fifo.ttl > 0) { + if (mutable_cf_options.compaction_options_fifo.ttl > 0) { c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer); } if (c == nullptr) { diff --git a/db/compaction_picker_test.cc b/db/compaction_picker_test.cc index bba2d073d..37c24841b 100644 --- a/db/compaction_picker_test.cc +++ b/db/compaction_picker_test.cc @@ -496,7 +496,7 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) { const uint64_t kMaxSize = kFileSize * kFileCount / 2; fifo_options_.max_table_files_size = kMaxSize; - ioptions_.compaction_options_fifo = fifo_options_; + mutable_cf_options_.compaction_options_fifo = fifo_options_; FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_); UpdateVersionStorageInfo(); // must return false when there's no files. diff --git a/db/db_options_test.cc b/db/db_options_test.cc index d2b2b2981..3cd4dcdc5 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -532,6 +532,114 @@ TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) { ASSERT_EQ(31 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate); } +TEST_F(DBOptionsTest, SetFIFOCompactionOptions) { + Options options; + options.compaction_style = kCompactionStyleFIFO; + options.write_buffer_size = 10 << 10; // 10KB + options.arena_block_size = 4096; + options.compression = kNoCompression; + options.create_if_missing = true; + options.compaction_options_fifo.allow_compaction = false; + env_->time_elapse_only_sleep_ = false; + options.env = env_; + + // Test dynamically changing compaction_options_fifo.ttl + env_->addon_time_.store(0); + options.compaction_options_fifo.ttl = 1 * 60 * 60; // 1 hour + ASSERT_OK(TryReopen(options)); + + Random rnd(301); + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980))); + } + Flush(); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Add 61 seconds to the time. + env_->addon_time_.fetch_add(61); + + // No files should be compacted as ttl is set to 1 hour. + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 3600); + dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Set ttl to 1 minute. So all files should get deleted. + ASSERT_OK(dbfull()->SetOptions({{"compaction_options_fifo", "{ttl=60;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 60); + dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 0); + + // Test dynamically changing compaction_options_fifo.max_table_files_size + env_->addon_time_.store(0); + options.compaction_options_fifo.max_table_files_size = 500 << 10; // 00KB + options.compaction_options_fifo.ttl = 0; + DestroyAndReopen(options); + + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980))); + } + Flush(); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // No files should be compacted as max_table_files_size is set to 500 KB. + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 500 << 10); + dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Set max_table_files_size to 12 KB. So only 1 file should remain now. + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{max_table_files_size=12288;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 12 << 10); + dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 1); + + // Test dynamically changing compaction_options_fifo.allow_compaction + options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB + options.compaction_options_fifo.ttl = 0; + options.compaction_options_fifo.allow_compaction = false; + options.level0_file_num_compaction_trigger = 6; + DestroyAndReopen(options); + + for (int i = 0; i < 10; i++) { + // Generate and flush a file about 10KB. + for (int j = 0; j < 10; j++) { + ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980))); + } + Flush(); + } + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // No files should be compacted as max_table_files_size is set to 500 KB and + // allow_compaction is false + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_EQ(NumTableFilesAtLevel(0), 10); + + // Set allow_compaction to true. So number of files should be between 1 and 5. + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{allow_compaction=true;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); + dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr); + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + ASSERT_GE(NumTableFilesAtLevel(0), 1); + ASSERT_LE(NumTableFilesAtLevel(0), 5); +} + #endif // ROCKSDB_LITE } // namespace rocksdb diff --git a/db/db_test.cc b/db/db_test.cc index 2e9a5075a..d3834c05c 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -4292,6 +4292,72 @@ TEST_F(DBTest, DynamicCompactionOptions) { } #endif // ROCKSDB_LITE +// Test dynamic FIFO copmaction options. +// This test covers just option parsing and makes sure that the options are +// correctly assigned. Also look at DBOptionsTest.SetFIFOCompactionOptions +// test which makes sure that the FIFO compaction funcionality is working +// as expected on dynamically changing the options. +// Even more FIFOCompactionTests are at DBTest.FIFOCompaction* . +TEST_F(DBTest, DynamicFIFOCompactionOptions) { + Options options; + options.create_if_missing = true; + DestroyAndReopen(options); + + // Initial defaults + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 1024 * 1024 * 1024); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 0); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{max_table_files_size=23;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 23); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 0); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + + ASSERT_OK(dbfull()->SetOptions({{"compaction_options_fifo", "{ttl=97}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 23); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 97); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + + ASSERT_OK(dbfull()->SetOptions({{"compaction_options_fifo", "{ttl=203;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 23); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 203); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + false); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{allow_compaction=true;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 23); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 203); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", "{max_table_files_size=31;ttl=19;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 31); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 19); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); + + ASSERT_OK(dbfull()->SetOptions( + {{"compaction_options_fifo", + "{max_table_files_size=51;ttl=49;allow_compaction=true;}"}})); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.max_table_files_size, + 51); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.ttl, 49); + ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction, + true); +} + TEST_F(DBTest, FileCreationRandomFailure) { Options options; options.env = env_; diff --git a/db/version_set.cc b/db/version_set.cc index 100d5a786..892f6d969 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -1328,6 +1328,7 @@ void VersionStorageInfo::EstimateCompactionBytesNeeded( namespace { uint32_t GetExpiredTtlFilesCount(const ImmutableCFOptions& ioptions, + const MutableCFOptions& mutable_cf_options, const std::vector& files) { uint32_t ttl_expired_files_count = 0; @@ -1341,8 +1342,8 @@ uint32_t GetExpiredTtlFilesCount(const ImmutableCFOptions& ioptions, auto creation_time = f->fd.table_reader->GetTableProperties()->creation_time; if (creation_time > 0 && - creation_time < - (current_time - ioptions.compaction_options_fifo.ttl)) { + creation_time < (current_time - + mutable_cf_options.compaction_options_fifo.ttl)) { ttl_expired_files_count++; } } @@ -1389,19 +1390,19 @@ void VersionStorageInfo::ComputeCompactionScore( } if (compaction_style_ == kCompactionStyleFIFO) { - score = - static_cast(total_size) / - immutable_cf_options.compaction_options_fifo.max_table_files_size; - if (immutable_cf_options.compaction_options_fifo.allow_compaction) { + score = static_cast(total_size) / + mutable_cf_options.compaction_options_fifo.max_table_files_size; + if (mutable_cf_options.compaction_options_fifo.allow_compaction) { score = std::max( static_cast(num_sorted_runs) / mutable_cf_options.level0_file_num_compaction_trigger, score); } - if (immutable_cf_options.compaction_options_fifo.ttl > 0) { - score = std::max(static_cast(GetExpiredTtlFilesCount( - immutable_cf_options, files_[level])), - score); + if (mutable_cf_options.compaction_options_fifo.ttl > 0) { + score = std::max( + static_cast(GetExpiredTtlFilesCount( + immutable_cf_options, mutable_cf_options, files_[level])), + score); } } else { diff --git a/include/rocksdb/advanced_options.h b/include/rocksdb/advanced_options.h index 06db9ef9f..99dcf7846 100644 --- a/include/rocksdb/advanced_options.h +++ b/include/rocksdb/advanced_options.h @@ -460,6 +460,10 @@ struct AdvancedColumnFamilyOptions { CompactionOptionsUniversal compaction_options_universal; // The options for FIFO compaction style + // + // Dynamically changeable through SetOptions() API + // Dynamic change example: + // SetOption("compaction_options_fifo", "{max_table_files_size=100;ttl=2;}") CompactionOptionsFIFO compaction_options_fifo; // An iteration->Next() sequentially skips over keys with the same diff --git a/options/cf_options.cc b/options/cf_options.cc index 67cbef68f..c2d7290f5 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -28,7 +28,6 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options, : compaction_style(cf_options.compaction_style), compaction_pri(cf_options.compaction_pri), compaction_options_universal(cf_options.compaction_options_universal), - compaction_options_fifo(cf_options.compaction_options_fifo), prefix_extractor(cf_options.prefix_extractor.get()), user_comparator(cf_options.comparator), internal_comparator(InternalKeyComparator(cf_options.comparator)), diff --git a/options/cf_options.h b/options/cf_options.h index f376729f8..35edf7c9c 100644 --- a/options/cf_options.h +++ b/options/cf_options.h @@ -31,7 +31,6 @@ struct ImmutableCFOptions { CompactionPri compaction_pri; CompactionOptionsUniversal compaction_options_universal; - CompactionOptionsFIFO compaction_options_fifo; const SliceTransform* prefix_extractor; @@ -149,6 +148,7 @@ struct MutableCFOptions { max_bytes_for_level_multiplier(options.max_bytes_for_level_multiplier), max_bytes_for_level_multiplier_additional( options.max_bytes_for_level_multiplier_additional), + compaction_options_fifo(options.compaction_options_fifo), max_sequential_skip_in_iterations( options.max_sequential_skip_in_iterations), paranoid_file_checks(options.paranoid_file_checks), @@ -176,6 +176,7 @@ struct MutableCFOptions { target_file_size_multiplier(0), max_bytes_for_level_base(0), max_bytes_for_level_multiplier(0), + compaction_options_fifo(), max_sequential_skip_in_iterations(0), paranoid_file_checks(false), report_bg_io_stats(false), @@ -222,6 +223,7 @@ struct MutableCFOptions { uint64_t max_bytes_for_level_base; double max_bytes_for_level_multiplier; std::vector max_bytes_for_level_multiplier_additional; + CompactionOptionsFIFO compaction_options_fifo; // Misc options uint64_t max_sequential_skip_in_iterations; diff --git a/options/options_helper.cc b/options/options_helper.cc index 8525f1464..c4d2beb8f 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -165,6 +165,8 @@ ColumnFamilyOptions BuildColumnFamilyOptions( cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value); } + cf_opts.compaction_options_fifo = mutable_cf_options.compaction_options_fifo; + // Misc options cf_opts.max_sequential_skip_in_iterations = mutable_cf_options.max_sequential_skip_in_iterations; @@ -181,6 +183,12 @@ ColumnFamilyOptions BuildColumnFamilyOptions( #ifndef ROCKSDB_LITE +template +Status GetStringFromStruct( + std::string* opt_string, const T& options, + const std::unordered_map type_info, + const std::string& delimiter); + namespace { template bool ParseEnum(const std::unordered_map& type_map, @@ -255,6 +263,76 @@ bool ParseVectorCompressionType( return true; } +// This is to handle backward compatibility, where compaction_options_fifo +// could be assigned a single scalar value, say, like "23", which would be +// assigned to max_table_files_size. +bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str, + CompactionOptionsFIFO* options) { + if (opt_str.find("=") != std::string::npos) { + // New format. Go do your new parsing using ParseStructOptions. + return false; + } + + // Old format. Parse just a single uint64_t value. + options->max_table_files_size = ParseUint64(opt_str); + return true; +} + +template +bool SerializeStruct( + const T& options, std::string* value, + std::unordered_map type_info_map) { + std::string opt_str; + Status s = GetStringFromStruct(&opt_str, options, type_info_map, ";"); + if (!s.ok()) { + return false; + } + *value = "{" + opt_str + "}"; + return true; +} + +template +bool ParseSingleStructOption( + const std::string& opt_val_str, T* options, + std::unordered_map type_info_map) { + size_t end = opt_val_str.find('='); + std::string key = opt_val_str.substr(0, end); + std::string value = opt_val_str.substr(end + 1); + auto iter = type_info_map.find(key); + if (iter == type_info_map.end()) { + return false; + } + const auto& opt_info = iter->second; + return ParseOptionHelper( + reinterpret_cast(options) + opt_info.mutable_offset, opt_info.type, + value); +} + +template +bool ParseStructOptions( + const std::string& opt_str, T* options, + std::unordered_map type_info_map) { + assert(!opt_str.empty()); + + size_t start = 0; + if (opt_str[0] == '{') { + start++; + } + while ((start != std::string::npos) && (start < opt_str.size())) { + if (opt_str[start] == '}') { + break; + } + size_t end = opt_str.find(';', start); + size_t len = (end == std::string::npos) ? end : end - start; + if (!ParseSingleStructOption(opt_str.substr(start, len), options, + type_info_map)) { + return false; + } + start = (end == std::string::npos) ? end : end + 1; + } + return true; +} + bool ParseSliceTransformHelper( const std::string& kFixedPrefixName, const std::string& kCappedPrefixName, const std::string& value, @@ -379,6 +457,15 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, return ParseEnum( info_log_level_string_map, value, reinterpret_cast(opt_address)); + case OptionType::kCompactionOptionsFIFO: { + if (!FIFOCompactionOptionsSpecialCase( + value, reinterpret_cast(opt_address))) { + return ParseStructOptions( + value, reinterpret_cast(opt_address), + fifo_compaction_options_type_info); + } + return true; + } default: return false; } @@ -543,6 +630,11 @@ bool SerializeSingleOptionHelper(const char* opt_address, return SerializeEnum( info_log_level_string_map, *reinterpret_cast(opt_address), value); + case OptionType::kCompactionOptionsFIFO: { + return SerializeStruct( + *reinterpret_cast(opt_address), value, + fifo_compaction_options_type_info); + } default: return false; } @@ -768,9 +860,6 @@ Status ParseColumnFamilyOption(const std::string& name, new_options->compression_opts.max_dict_bytes = ParseInt(value.substr(start, value.size() - start)); } - } else if (name == "compaction_options_fifo") { - new_options->compaction_options_fifo.max_table_files_size = - ParseUint64(value); } else { auto iter = cf_options_type_info.find(name); if (iter == cf_options_type_info.end()) { @@ -805,17 +894,18 @@ Status ParseColumnFamilyOption(const std::string& name, return Status::OK(); } -bool SerializeSingleDBOption(std::string* opt_string, - const DBOptions& db_options, - const std::string& name, - const std::string& delimiter) { - auto iter = db_options_type_info.find(name); - if (iter == db_options_type_info.end()) { +template +bool SerializeSingleStructOption( + std::string* opt_string, const T& options, + const std::unordered_map type_info, + const std::string& name, const std::string& delimiter) { + auto iter = type_info.find(name); + if (iter == type_info.end()) { return false; } auto& opt_info = iter->second; const char* opt_address = - reinterpret_cast(&db_options) + opt_info.offset; + reinterpret_cast(&options) + opt_info.offset; std::string value; bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value); if (result) { @@ -824,72 +914,45 @@ bool SerializeSingleDBOption(std::string* opt_string, return result; } -Status GetStringFromDBOptions(std::string* opt_string, - const DBOptions& db_options, - const std::string& delimiter) { +template +Status GetStringFromStruct( + std::string* opt_string, const T& options, + const std::unordered_map type_info, + const std::string& delimiter) { assert(opt_string); opt_string->clear(); - for (auto iter = db_options_type_info.begin(); - iter != db_options_type_info.end(); ++iter) { + for (auto iter = type_info.begin(); iter != type_info.end(); ++iter) { if (iter->second.verification == OptionVerificationType::kDeprecated) { // If the option is no longer used in rocksdb and marked as deprecated, // we skip it in the serialization. continue; } std::string single_output; - bool result = SerializeSingleDBOption(&single_output, db_options, - iter->first, delimiter); - assert(result); + bool result = SerializeSingleStructOption( + &single_output, options, type_info, iter->first, delimiter); if (result) { opt_string->append(single_output); + } else { + return Status::InvalidArgument("failed to serialize %s\n", + iter->first.c_str()); } + assert(result); } return Status::OK(); } -bool SerializeSingleColumnFamilyOption(std::string* opt_string, - const ColumnFamilyOptions& cf_options, - const std::string& name, - const std::string& delimiter) { - auto iter = cf_options_type_info.find(name); - if (iter == cf_options_type_info.end()) { - return false; - } - auto& opt_info = iter->second; - const char* opt_address = - reinterpret_cast(&cf_options) + opt_info.offset; - std::string value; - bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value); - if (result) { - *opt_string = name + "=" + value + delimiter; - } - return result; +Status GetStringFromDBOptions(std::string* opt_string, + const DBOptions& db_options, + const std::string& delimiter) { + return GetStringFromStruct(opt_string, db_options, + db_options_type_info, delimiter); } Status GetStringFromColumnFamilyOptions(std::string* opt_string, const ColumnFamilyOptions& cf_options, const std::string& delimiter) { - assert(opt_string); - opt_string->clear(); - for (auto iter = cf_options_type_info.begin(); - iter != cf_options_type_info.end(); ++iter) { - if (iter->second.verification == OptionVerificationType::kDeprecated) { - // If the option is no longer used in rocksdb and marked as deprecated, - // we skip it in the serialization. - continue; - } - std::string single_output; - bool result = SerializeSingleColumnFamilyOption(&single_output, cf_options, - iter->first, delimiter); - if (result) { - opt_string->append(single_output); - } else { - return Status::InvalidArgument("failed to serialize %s\n", - iter->first.c_str()); - } - assert(result); - } - return Status::OK(); + return GetStringFromStruct( + opt_string, cf_options, cf_options_type_info, delimiter); } Status GetStringFromCompressionType(std::string* compression_str, diff --git a/options/options_helper.h b/options/options_helper.h index 49ab09f5a..8993daeef 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -82,6 +82,7 @@ enum class OptionType { kComparator, kCompactionFilter, kCompactionFilterFactory, + kCompactionOptionsFIFO, kMergeOperator, kMemTableRepFactory, kBlockBasedTableIndexType, @@ -587,7 +588,26 @@ static std::unordered_map cf_options_type_info = { OptionType::kCompactionStyle, OptionVerificationType::kNormal, false, 0}}, {"compaction_pri", {offset_of(&ColumnFamilyOptions::compaction_pri), - OptionType::kCompactionPri, OptionVerificationType::kNormal, false, 0}}}; + OptionType::kCompactionPri, OptionVerificationType::kNormal, false, 0}}, + {"compaction_options_fifo", + {offset_of(&ColumnFamilyOptions::compaction_options_fifo), + OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal, true, + offsetof(struct MutableCFOptions, compaction_options_fifo)}}}; + +static std::unordered_map + fifo_compaction_options_type_info = { + {"max_table_files_size", + {offset_of(&CompactionOptionsFIFO::max_table_files_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, true, + offsetof(struct CompactionOptionsFIFO, max_table_files_size)}}, + {"ttl", + {offset_of(&CompactionOptionsFIFO::ttl), OptionType::kUInt64T, + OptionVerificationType::kNormal, true, + offsetof(struct CompactionOptionsFIFO, ttl)}}, + {"allow_compaction", + {offset_of(&CompactionOptionsFIFO::allow_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, true, + offsetof(struct CompactionOptionsFIFO, allow_compaction)}}}; static std::unordered_map compression_type_string_map = { diff --git a/options/options_parser.cc b/options/options_parser.cc index 3bbf8a737..f0eaa8045 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -587,6 +587,17 @@ bool AreEqualOptions( case OptionType::kInfoLogLevel: return (*reinterpret_cast(offset1) == *reinterpret_cast(offset2)); + case OptionType::kCompactionOptionsFIFO: { + CompactionOptionsFIFO lhs = + *reinterpret_cast(offset1); + CompactionOptionsFIFO rhs = + *reinterpret_cast(offset2); + if (lhs.max_table_files_size == rhs.max_table_files_size && + lhs.ttl == rhs.ttl && lhs.allow_compaction == rhs.allow_compaction) { + return true; + } + return false; + } default: if (type_info.verification == OptionVerificationType::kByName || type_info.verification == diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index 73008bbe4..33e9d4c18 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -371,7 +371,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { options->hard_rate_limit = 0; options->soft_rate_limit = 0; options->purge_redundant_kvs_while_flush = false; - options->compaction_options_fifo = CompactionOptionsFIFO(); options->max_mem_compaction_level = 0; char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)]; @@ -427,7 +426,8 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { "compaction_pri=kMinOverlappingRatio;" "hard_pending_compaction_bytes_limit=0;" "disable_auto_compactions=false;" - "report_bg_io_stats=true;", + "report_bg_io_stats=true;" + "compaction_options_fifo={max_table_files_size=3;ttl=100;allow_compaction=false;};", new_options)); ASSERT_EQ(unset_bytes_base, diff --git a/util/testutil.cc b/util/testutil.cc index f3010f3f2..ecc4cbe14 100644 --- a/util/testutil.cc +++ b/util/testutil.cc @@ -310,6 +310,7 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) { cf_opt->paranoid_file_checks = rnd->Uniform(2); cf_opt->purge_redundant_kvs_while_flush = rnd->Uniform(2); cf_opt->force_consistency_checks = rnd->Uniform(2); + cf_opt->compaction_options_fifo.allow_compaction = rnd->Uniform(2); // double options cf_opt->hard_rate_limit = static_cast(rnd->Uniform(10000)) / 13; @@ -352,6 +353,9 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) { cf_opt->target_file_size_base = uint_max + rnd->Uniform(10000); cf_opt->max_compaction_bytes = cf_opt->target_file_size_base * rnd->Uniform(100); + cf_opt->compaction_options_fifo.max_table_files_size = + uint_max + rnd->Uniform(10000); + cf_opt->compaction_options_fifo.ttl = uint_max + rnd->Uniform(10000); // unsigned int options cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000);