From 259b6ec8da28cf02bd140040d9d832ee3cdd250f Mon Sep 17 00:00:00 2001 From: mrambacher Date: Fri, 3 Apr 2020 10:48:46 -0700 Subject: [PATCH] Move the OptionTypeMap code closer to home (#6198) Summary: This is a predecessor to the Configurable PR. This change moves the OptionTypeInfo maps closer to where they will be used. When the Configurable changes are adopted, these values will become static and not associated with the OptionsHelper. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6198 Reviewed By: siying Differential Revision: D20778108 Pulled By: zhichao-cao fbshipit-source-id: a9f85fc73bc53503656e1958ecc1e764052fd1aa --- db/db_options_test.cc | 6 +- options/cf_options.cc | 437 +++++++++ options/db_options.cc | 316 +++++++ options/options_helper.cc | 842 ++---------------- options/options_helper.h | 78 +- options/options_parser.cc | 24 +- options/options_type.h | 141 +++ .../block_based/block_based_table_factory.cc | 170 +++- table/block_based/block_based_table_factory.h | 98 -- table/plain/plain_table_factory.cc | 37 +- table/plain/plain_table_factory.h | 25 - 11 files changed, 1147 insertions(+), 1027 deletions(-) create mode 100644 options/options_type.h diff --git a/db/db_options_test.cc b/db/db_options_test.cc index 29db9d095..fc404f112 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -38,8 +38,7 @@ class DBOptionsTest : public DBTestBase { StringToMap(options_str, &options_map); std::unordered_map mutable_map; for (const auto opt : db_options_type_info) { - if (opt.second.is_mutable && - opt.second.verification != OptionVerificationType::kDeprecated) { + if (opt.second.IsMutable() && !opt.second.IsDeprecated()) { mutable_map[opt.first] = options_map[opt.first]; } } @@ -54,8 +53,7 @@ class DBOptionsTest : public DBTestBase { StringToMap(options_str, &options_map); std::unordered_map mutable_map; for (const auto opt : cf_options_type_info) { - if (opt.second.is_mutable && - opt.second.verification != OptionVerificationType::kDeprecated) { + if (opt.second.IsMutable() && !opt.second.IsDeprecated()) { mutable_map[opt.first] = options_map[opt.first]; } } diff --git a/options/cf_options.cc b/options/cf_options.cc index 37aa3de2c..65b198510 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -9,14 +9,451 @@ #include #include #include + #include "options/db_options.h" +#include "options/options_helper.h" #include "port/port.h" #include "rocksdb/concurrent_task_limiter.h" +#include "rocksdb/convenience.h" #include "rocksdb/env.h" #include "rocksdb/file_system.h" +#include "rocksdb/merge_operator.h" #include "rocksdb/options.h" +#include "rocksdb/utilities/object_registry.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/plain/plain_table_factory.h" +#include "util/cast_util.h" namespace ROCKSDB_NAMESPACE { +// offset_of is used to get the offset of a class data member +// ex: offset_of(&ColumnFamilyOptions::num_levels) +// This call will return the offset of num_levels in ColumnFamilyOptions class +// +// This is the same as offsetof() but allow us to work with non standard-layout +// classes and structures +// refs: +// http://en.cppreference.com/w/cpp/concept/StandardLayoutType +// https://gist.github.com/graphitemaster/494f21190bb2c63c5516 +#ifndef ROCKSDB_LITE +ColumnFamilyOptions OptionsHelper::dummy_cf_options; +template +int offset_of(T1 ColumnFamilyOptions::*member) { + return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - + size_t(&OptionsHelper::dummy_cf_options)); +} +template +int offset_of(T1 AdvancedColumnFamilyOptions::*member) { + return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - + size_t(&OptionsHelper::dummy_cf_options)); +} + +const std::string kNameComparator = "comparator"; +const std::string kNameMergeOperator = "merge_operator"; + +std::unordered_map + OptionsHelper::cf_options_type_info = { + /* not yet supported + CompressionOptions compression_opts; + TablePropertiesCollectorFactories table_properties_collector_factories; + typedef std::vector> + TablePropertiesCollectorFactories; + UpdateStatus (*inplace_callback)(char* existing_value, + uint34_t* existing_value_size, + Slice delta_value, + std::string* merged_value); + std::vector cf_paths; + */ + {"report_bg_io_stats", + {offset_of(&ColumnFamilyOptions::report_bg_io_stats), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, report_bg_io_stats)}}, + {"compaction_measure_io_stats", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"disable_auto_compactions", + {offset_of(&ColumnFamilyOptions::disable_auto_compactions), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, disable_auto_compactions)}}, + {"filter_deletes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"inplace_update_support", + {offset_of(&ColumnFamilyOptions::inplace_update_support), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"level_compaction_dynamic_level_bytes", + {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"optimize_filters_for_hits", + {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"paranoid_file_checks", + {offset_of(&ColumnFamilyOptions::paranoid_file_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, paranoid_file_checks)}}, + {"force_consistency_checks", + {offset_of(&ColumnFamilyOptions::force_consistency_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"purge_redundant_kvs_while_flush", + {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush), + OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"verify_checksums_in_compaction", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"soft_pending_compaction_bytes_limit", + {offset_of(&ColumnFamilyOptions::soft_pending_compaction_bytes_limit), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + soft_pending_compaction_bytes_limit)}}, + {"hard_pending_compaction_bytes_limit", + {offset_of(&ColumnFamilyOptions::hard_pending_compaction_bytes_limit), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + hard_pending_compaction_bytes_limit)}}, + {"hard_rate_limit", + {0, OptionType::kDouble, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"soft_rate_limit", + {0, OptionType::kDouble, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"max_compaction_bytes", + {offset_of(&ColumnFamilyOptions::max_compaction_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_compaction_bytes)}}, + {"expanded_compaction_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"level0_file_num_compaction_trigger", + {offset_of(&ColumnFamilyOptions::level0_file_num_compaction_trigger), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + level0_file_num_compaction_trigger)}}, + {"level0_slowdown_writes_trigger", + {offset_of(&ColumnFamilyOptions::level0_slowdown_writes_trigger), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger)}}, + {"level0_stop_writes_trigger", + {offset_of(&ColumnFamilyOptions::level0_stop_writes_trigger), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, level0_stop_writes_trigger)}}, + {"max_grandparent_overlap_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"max_mem_compaction_level", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"max_write_buffer_number", + {offset_of(&ColumnFamilyOptions::max_write_buffer_number), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_write_buffer_number)}}, + {"max_write_buffer_number_to_maintain", + {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_write_buffer_size_to_maintain", + {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain), + OptionType::kInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"min_write_buffer_number_to_merge", + {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"num_levels", + {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"source_compaction_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"target_file_size_multiplier", + {offset_of(&ColumnFamilyOptions::target_file_size_multiplier), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, target_file_size_multiplier)}}, + {"arena_block_size", + {offset_of(&ColumnFamilyOptions::arena_block_size), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, arena_block_size)}}, + {"inplace_update_num_locks", + {offset_of(&ColumnFamilyOptions::inplace_update_num_locks), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, inplace_update_num_locks)}}, + {"max_successive_merges", + {offset_of(&ColumnFamilyOptions::max_successive_merges), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_successive_merges)}}, + {"memtable_huge_page_size", + {offset_of(&ColumnFamilyOptions::memtable_huge_page_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, memtable_huge_page_size)}}, + {"memtable_prefix_bloom_huge_page_tlb_size", + {0, OptionType::kSizeT, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"write_buffer_size", + {offset_of(&ColumnFamilyOptions::write_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, write_buffer_size)}}, + {"bloom_locality", + {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"memtable_prefix_bloom_bits", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"memtable_prefix_bloom_size_ratio", + {offset_of(&ColumnFamilyOptions::memtable_prefix_bloom_size_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio)}}, + {"memtable_prefix_bloom_probes", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"memtable_whole_key_filtering", + {offset_of(&ColumnFamilyOptions::memtable_whole_key_filtering), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, memtable_whole_key_filtering)}}, + {"min_partial_merge_operands", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"max_bytes_for_level_base", + {offset_of(&ColumnFamilyOptions::max_bytes_for_level_base), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_bytes_for_level_base)}}, + {"snap_refresh_nanos", + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"max_bytes_for_level_multiplier", + {offset_of(&ColumnFamilyOptions::max_bytes_for_level_multiplier), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}}, + {"max_bytes_for_level_multiplier_additional", + {offset_of( + &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional), + OptionType::kVectorInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + max_bytes_for_level_multiplier_additional)}}, + {"max_sequential_skip_in_iterations", + {offset_of(&ColumnFamilyOptions::max_sequential_skip_in_iterations), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + max_sequential_skip_in_iterations)}}, + {"target_file_size_base", + {offset_of(&ColumnFamilyOptions::target_file_size_base), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, target_file_size_base)}}, + {"rate_limit_delay_max_milliseconds", + {0, OptionType::kUInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"compression", + {offset_of(&ColumnFamilyOptions::compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, compression)}}, + {"compression_per_level", + {offset_of(&ColumnFamilyOptions::compression_per_level), + OptionType::kVectorCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"bottommost_compression", + {offset_of(&ColumnFamilyOptions::bottommost_compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, bottommost_compression)}}, + {kNameComparator, + {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator, + OptionVerificationType::kByName, OptionTypeFlags::kNone, 0}}, + {"prefix_extractor", + {offset_of(&ColumnFamilyOptions::prefix_extractor), + OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, prefix_extractor)}}, + {"memtable_insert_with_hint_prefix_extractor", + {offset_of( + &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), + OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, + OptionTypeFlags::kNone, 0}}, + {"memtable_factory", + {offset_of(&ColumnFamilyOptions::memtable_factory), + OptionType::kMemTableRepFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"table_factory", + {offset_of(&ColumnFamilyOptions::table_factory), + OptionType::kTableFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"compaction_filter", + {offset_of(&ColumnFamilyOptions::compaction_filter), + OptionType::kCompactionFilter, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"compaction_filter_factory", + {offset_of(&ColumnFamilyOptions::compaction_filter_factory), + OptionType::kCompactionFilterFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {kNameMergeOperator, + {offset_of(&ColumnFamilyOptions::merge_operator), + OptionType::kMergeOperator, + OptionVerificationType::kByNameAllowFromNull, OptionTypeFlags::kNone, + 0}}, + {"compaction_style", + {offset_of(&ColumnFamilyOptions::compaction_style), + OptionType::kCompactionStyle, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"compaction_pri", + {offset_of(&ColumnFamilyOptions::compaction_pri), + OptionType::kCompactionPri, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"compaction_options_fifo", + {offset_of(&ColumnFamilyOptions::compaction_options_fifo), + OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, compaction_options_fifo)}}, + {"compaction_options_universal", + {offset_of(&ColumnFamilyOptions::compaction_options_universal), + OptionType::kCompactionOptionsUniversal, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, compaction_options_universal)}}, + {"ttl", + {offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, ttl)}}, + {"periodic_compaction_seconds", + {offset_of(&ColumnFamilyOptions::periodic_compaction_seconds), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, periodic_compaction_seconds)}}, + {"sample_for_compression", + {offset_of(&ColumnFamilyOptions::sample_for_compression), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, sample_for_compression)}}}; + +Status ParseColumnFamilyOption(const std::string& name, + const std::string& org_value, + ColumnFamilyOptions* new_options, + bool input_strings_escaped) { + const std::string& value = + input_strings_escaped ? UnescapeOptionString(org_value) : org_value; + try { + if (name == "block_based_table_factory") { + // Nested options + BlockBasedTableOptions table_opt, base_table_options; + BlockBasedTableFactory* block_based_table_factory = + static_cast_with_check( + new_options->table_factory.get()); + if (block_based_table_factory != nullptr) { + base_table_options = block_based_table_factory->table_options(); + } + Status table_opt_s = GetBlockBasedTableOptionsFromString( + base_table_options, value, &table_opt); + if (!table_opt_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt)); + } else if (name == "plain_table_factory") { + // Nested options + PlainTableOptions table_opt, base_table_options; + PlainTableFactory* plain_table_factory = + static_cast_with_check( + new_options->table_factory.get()); + if (plain_table_factory != nullptr) { + base_table_options = plain_table_factory->table_options(); + } + Status table_opt_s = + GetPlainTableOptionsFromString(base_table_options, value, &table_opt); + if (!table_opt_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->table_factory.reset(NewPlainTableFactory(table_opt)); + } else if (name == "memtable") { + std::unique_ptr new_mem_factory; + Status mem_factory_s = + GetMemTableRepFactoryFromString(value, &new_mem_factory); + if (!mem_factory_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->memtable_factory.reset(new_mem_factory.release()); + } else if (name == "bottommost_compression_opts") { + Status s = ParseCompressionOptions( + value, name, new_options->bottommost_compression_opts); + if (!s.ok()) { + return s; + } + } else if (name == "compression_opts") { + Status s = + ParseCompressionOptions(value, name, new_options->compression_opts); + if (!s.ok()) { + return s; + } + } else { + if (name == kNameComparator) { + // Try to get comparator from object registry first. + // Only support static comparator for now. + Status status = ObjectRegistry::NewInstance()->NewStaticObject( + value, &new_options->comparator); + if (status.ok()) { + return status; + } + } else if (name == kNameMergeOperator) { + // Try to get merge operator from object registry first. + std::shared_ptr mo; + Status status = + ObjectRegistry::NewInstance()->NewSharedObject( + value, &new_options->merge_operator); + // Only support static comparator for now. + if (status.ok()) { + return status; + } + } + + auto iter = cf_options_type_info.find(name); + if (iter == cf_options_type_info.end()) { + return Status::InvalidArgument( + "Unable to parse the specified CF option " + name); + } + const auto& opt_info = iter->second; + if (opt_info.IsDeprecated() || + ParseOptionHelper( + reinterpret_cast(new_options) + opt_info.offset, + opt_info.type, value)) { + return Status::OK(); + } else if (opt_info.IsByName()) { + return Status::NotSupported("Deserializing the specified CF option " + + name + " is not supported"); + } else { + return Status::InvalidArgument( + "Unable to parse the specified CF option " + name); + } + } + } catch (const std::exception&) { + return Status::InvalidArgument("unable to parse the specified option " + + name); + } + return Status::OK(); +} +#endif // ROCKSDB_LITE ImmutableCFOptions::ImmutableCFOptions(const Options& options) : ImmutableCFOptions(ImmutableDBOptions(options), options) {} diff --git a/options/db_options.cc b/options/db_options.cc index 8ef147c5e..227525fc7 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -9,6 +9,7 @@ #include "db/version_edit.h" #include "logging/logging.h" +#include "options/options_helper.h" #include "port/port.h" #include "rocksdb/cache.h" #include "rocksdb/env.h" @@ -17,6 +18,321 @@ #include "rocksdb/wal_filter.h" namespace ROCKSDB_NAMESPACE { +#ifndef ROCKSDB_LITE +std::unordered_map + OptionsHelper::db_options_type_info = { + /* + // not yet supported + std::shared_ptr row_cache; + std::shared_ptr delete_scheduler; + std::shared_ptr info_log; + std::shared_ptr rate_limiter; + std::shared_ptr statistics; + std::vector db_paths; + std::vector> listeners; + */ + {"advise_random_on_open", + {offsetof(struct DBOptions, advise_random_on_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"allow_mmap_reads", + {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"allow_fallocate", + {offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"allow_mmap_writes", + {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"use_direct_reads", + {offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"use_direct_writes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"use_direct_io_for_flush_and_compaction", + {offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"allow_2pc", + {offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"allow_os_buffer", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"create_if_missing", + {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"create_missing_column_families", + {offsetof(struct DBOptions, create_missing_column_families), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"disableDataSync", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"disable_data_sync", // for compatibility + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"enable_thread_tracking", + {offsetof(struct DBOptions, enable_thread_tracking), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"error_if_exists", + {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"is_fd_close_on_exec", + {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"paranoid_checks", + {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"skip_log_error_on_recovery", + {offsetof(struct DBOptions, skip_log_error_on_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"skip_stats_update_on_db_open", + {offsetof(struct DBOptions, skip_stats_update_on_db_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"skip_checking_sst_file_sizes_on_db_open", + {offsetof(struct DBOptions, skip_checking_sst_file_sizes_on_db_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"new_table_reader_for_compaction_inputs", + {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"compaction_readahead_size", + {offsetof(struct DBOptions, compaction_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, compaction_readahead_size)}}, + {"random_access_max_buffer_size", + {offsetof(struct DBOptions, random_access_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"use_adaptive_mutex", + {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"use_fsync", + {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"max_background_jobs", + {offsetof(struct DBOptions, max_background_jobs), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, max_background_jobs)}}, + {"max_background_compactions", + {offsetof(struct DBOptions, max_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, max_background_compactions)}}, + {"base_background_compactions", + {offsetof(struct DBOptions, base_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, base_background_compactions)}}, + {"max_background_flushes", + {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"max_file_opening_threads", + {offsetof(struct DBOptions, max_file_opening_threads), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_open_files", + {offsetof(struct DBOptions, max_open_files), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, max_open_files)}}, + {"table_cache_numshardbits", + {offsetof(struct DBOptions, table_cache_numshardbits), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"db_write_buffer_size", + {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"keep_log_file_num", + {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"recycle_log_file_num", + {offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"log_file_time_to_roll", + {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"manifest_preallocation_size", + {offsetof(struct DBOptions, manifest_preallocation_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_log_file_size", + {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"db_log_dir", + {offsetof(struct DBOptions, db_log_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"wal_dir", + {offsetof(struct DBOptions, wal_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"max_subcompactions", + {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"WAL_size_limit_MB", + {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"WAL_ttl_seconds", + {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"bytes_per_sync", + {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, bytes_per_sync)}}, + {"delayed_write_rate", + {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, delayed_write_rate)}}, + {"delete_obsolete_files_period_micros", + {offsetof(struct DBOptions, delete_obsolete_files_period_micros), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, + delete_obsolete_files_period_micros)}}, + {"max_manifest_file_size", + {offsetof(struct DBOptions, max_manifest_file_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_total_wal_size", + {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, max_total_wal_size)}}, + {"wal_bytes_per_sync", + {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, wal_bytes_per_sync)}}, + {"strict_bytes_per_sync", + {offsetof(struct DBOptions, strict_bytes_per_sync), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, strict_bytes_per_sync)}}, + {"stats_dump_period_sec", + {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, stats_dump_period_sec)}}, + {"stats_persist_period_sec", + {offsetof(struct DBOptions, stats_persist_period_sec), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, stats_persist_period_sec)}}, + {"persist_stats_to_disk", + {offsetof(struct DBOptions, persist_stats_to_disk), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}}, + {"stats_history_buffer_size", + {offsetof(struct DBOptions, stats_history_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, stats_history_buffer_size)}}, + {"fail_if_options_file_error", + {offsetof(struct DBOptions, fail_if_options_file_error), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"enable_pipelined_write", + {offsetof(struct DBOptions, enable_pipelined_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"unordered_write", + {offsetof(struct DBOptions, unordered_write), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"allow_concurrent_memtable_write", + {offsetof(struct DBOptions, allow_concurrent_memtable_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"wal_recovery_mode", + {offsetof(struct DBOptions, wal_recovery_mode), + OptionType::kWALRecoveryMode, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"enable_write_thread_adaptive_yield", + {offsetof(struct DBOptions, enable_write_thread_adaptive_yield), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"write_thread_slow_yield_usec", + {offsetof(struct DBOptions, write_thread_slow_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_write_batch_group_size_bytes", + {offsetof(struct DBOptions, max_write_batch_group_size_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"write_thread_max_yield_usec", + {offsetof(struct DBOptions, write_thread_max_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"access_hint_on_compaction_start", + {offsetof(struct DBOptions, access_hint_on_compaction_start), + OptionType::kAccessHint, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"info_log_level", + {offsetof(struct DBOptions, info_log_level), OptionType::kInfoLogLevel, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"dump_malloc_stats", + {offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"avoid_flush_during_recovery", + {offsetof(struct DBOptions, avoid_flush_during_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"avoid_flush_during_shutdown", + {offsetof(struct DBOptions, avoid_flush_during_shutdown), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}}, + {"writable_file_max_buffer_size", + {offsetof(struct DBOptions, writable_file_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}}, + {"allow_ingest_behind", + {offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, allow_ingest_behind)}}, + {"preserve_deletes", + {offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, preserve_deletes)}}, + {"concurrent_prepare", // Deprecated by two_write_queues + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"two_write_queues", + {offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, two_write_queues)}}, + {"manual_wal_flush", + {offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, manual_wal_flush)}}, + {"seq_per_batch", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"atomic_flush", + {offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, atomic_flush)}}, + {"avoid_unnecessary_blocking_io", + {offsetof(struct DBOptions, avoid_unnecessary_blocking_io), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}}, + {"write_dbid_to_manifest", + {offsetof(struct DBOptions, write_dbid_to_manifest), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"log_readahead_size", + {offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"best_efforts_recovery", + {offsetof(struct DBOptions, best_efforts_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, +}; +#endif // ROCKSDB_LITE ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {} diff --git a/options/options_helper.cc b/options/options_helper.cc index 4a9c0fe96..f3ee13ad8 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -23,7 +23,6 @@ #include "rocksdb/utilities/object_registry.h" #include "table/block_based/block_based_table_factory.h" #include "table/plain/plain_table_factory.h" -#include "util/cast_util.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { @@ -256,18 +255,10 @@ std::unordered_map {"kDisableCompressionOption", kDisableCompressionOption}}; #ifndef ROCKSDB_LITE -const std::string kNameComparator = "comparator"; const std::string kNameEnv = "env"; -const std::string kNameMergeOperator = "merge_operator"; const std::string kOptNameBMCompOpts = "bottommost_compression_opts"; const std::string kOptNameCompOpts = "compression_opts"; -template -Status GetStringFromStruct( - std::string* opt_string, const T& options, - const std::unordered_map& type_info, - const std::string& delimiter); - namespace { template bool ParseEnum(const std::unordered_map& type_map, @@ -357,9 +348,8 @@ bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str, return true; } -template -bool SerializeStruct( - const T& options, std::string* value, +static bool SerializeStruct( + const void* const options, std::string* value, const std::unordered_map& type_info_map) { std::string opt_str; Status s = GetStringFromStruct(&opt_str, options, type_info_map, ";"); @@ -370,9 +360,8 @@ bool SerializeStruct( return true; } -template -bool ParseSingleStructOption( - const std::string& opt_val_str, T* options, +static bool ParseSingleStructOption( + const std::string& opt_val_str, void* options, const std::unordered_map& type_info_map) { size_t end = opt_val_str.find('='); std::string key = opt_val_str.substr(0, end); @@ -382,7 +371,7 @@ bool ParseSingleStructOption( return false; } const auto& opt_info = iter->second; - if (opt_info.verification == OptionVerificationType::kDeprecated) { + if (opt_info.IsDeprecated()) { // Should also skip deprecated sub-options such as // fifo_compaction_options_type_info.ttl return true; @@ -392,9 +381,8 @@ bool ParseSingleStructOption( value); } -template -bool ParseStructOptions( - const std::string& opt_str, T* options, +static bool ParseStructOptions( + const std::string& opt_str, void* options, const std::unordered_map& type_info_map) { assert(!opt_str.empty()); @@ -565,21 +553,18 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, case OptionType::kCompactionOptionsFIFO: { if (!FIFOCompactionOptionsSpecialCase( value, reinterpret_cast(opt_address))) { - return ParseStructOptions( - value, reinterpret_cast(opt_address), - fifo_compaction_options_type_info); + return ParseStructOptions(value, opt_address, + fifo_compaction_options_type_info); } return true; } case OptionType::kLRUCacheOptions: { - return ParseStructOptions(value, - reinterpret_cast(opt_address), - lru_cache_options_type_info); + return ParseStructOptions(value, opt_address, + lru_cache_options_type_info); } case OptionType::kCompactionOptionsUniversal: - return ParseStructOptions( - value, reinterpret_cast(opt_address), - universal_compaction_options_type_info); + return ParseStructOptions(value, opt_address, + universal_compaction_options_type_info); case OptionType::kCompactionStopStyle: return ParseEnum( compaction_stop_style_string_map, value, @@ -771,13 +756,11 @@ bool SerializeSingleOptionHelper(const char* opt_address, info_log_level_string_map, *reinterpret_cast(opt_address), value); case OptionType::kCompactionOptionsFIFO: - return SerializeStruct( - *reinterpret_cast(opt_address), value, - fifo_compaction_options_type_info); + return SerializeStruct(opt_address, value, + fifo_compaction_options_type_info); case OptionType::kCompactionOptionsUniversal: - return SerializeStruct( - *reinterpret_cast(opt_address), - value, universal_compaction_options_type_info); + return SerializeStruct(opt_address, value, + universal_compaction_options_type_info); case OptionType::kCompactionStopStyle: return SerializeEnum( compaction_stop_style_string_map, @@ -889,11 +872,11 @@ Status GetMutableOptionsFromStrings( return Status::InvalidArgument("Unrecognized option: " + option_name); } const auto& opt_info = iter->second; - if (!opt_info.is_mutable) { + if (!opt_info.IsMutable()) { return Status::InvalidArgument("Option not changeable: " + option_name); } - if (opt_info.verification == OptionVerificationType::kDeprecated) { + if (opt_info.IsDeprecated()) { // log warning when user tries to set a deprecated option but don't // fail the call for compatibility. ROCKS_LOG_WARN(info_log, @@ -929,7 +912,7 @@ Status GetMutableDBOptionsFromStrings( return Status::InvalidArgument("Unrecognized option: " + o.first); } const auto& opt_info = iter->second; - if (!opt_info.is_mutable) { + if (!opt_info.IsMutable()) { return Status::InvalidArgument("Option not changeable: " + o.first); } bool is_ok = ParseOptionHelper( @@ -1022,163 +1005,30 @@ Status StringToMap(const std::string& opts_str, return Status::OK(); } -Status ParseColumnFamilyOption(const std::string& name, - const std::string& org_value, - ColumnFamilyOptions* new_options, - bool input_strings_escaped = false) { - const std::string& value = - input_strings_escaped ? UnescapeOptionString(org_value) : org_value; - try { - if (name == "block_based_table_factory") { - // Nested options - BlockBasedTableOptions table_opt, base_table_options; - BlockBasedTableFactory* block_based_table_factory = - static_cast_with_check( - new_options->table_factory.get()); - if (block_based_table_factory != nullptr) { - base_table_options = block_based_table_factory->table_options(); - } - Status table_opt_s = GetBlockBasedTableOptionsFromString( - base_table_options, value, &table_opt); - if (!table_opt_s.ok()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt)); - } else if (name == "plain_table_factory") { - // Nested options - PlainTableOptions table_opt, base_table_options; - PlainTableFactory* plain_table_factory = - static_cast_with_check( - new_options->table_factory.get()); - if (plain_table_factory != nullptr) { - base_table_options = plain_table_factory->table_options(); - } - Status table_opt_s = GetPlainTableOptionsFromString( - base_table_options, value, &table_opt); - if (!table_opt_s.ok()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - new_options->table_factory.reset(NewPlainTableFactory(table_opt)); - } else if (name == "memtable") { - std::unique_ptr new_mem_factory; - Status mem_factory_s = - GetMemTableRepFactoryFromString(value, &new_mem_factory); - if (!mem_factory_s.ok()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - new_options->memtable_factory.reset(new_mem_factory.release()); - } else if (name == "bottommost_compression_opts") { - Status s = ParseCompressionOptions( - value, name, new_options->bottommost_compression_opts); - if (!s.ok()) { - return s; - } - } else if (name == "compression_opts") { - Status s = - ParseCompressionOptions(value, name, new_options->compression_opts); - if (!s.ok()) { - return s; - } - } else { - if (name == kNameComparator) { - // Try to get comparator from object registry first. - // Only support static comparator for now. - Status status = ObjectRegistry::NewInstance()->NewStaticObject( - value, &new_options->comparator); - if (status.ok()) { - return status; - } - } else if (name == kNameMergeOperator) { - // Try to get merge operator from object registry first. - std::shared_ptr mo; - Status status = - ObjectRegistry::NewInstance()->NewSharedObject( - value, &new_options->merge_operator); - // Only support static comparator for now. - if (status.ok()) { - return status; - } - } - - auto iter = cf_options_type_info.find(name); - if (iter == cf_options_type_info.end()) { - return Status::InvalidArgument( - "Unable to parse the specified CF option " + name); - } - const auto& opt_info = iter->second; - if (opt_info.verification != OptionVerificationType::kDeprecated && - ParseOptionHelper( - reinterpret_cast(new_options) + opt_info.offset, - opt_info.type, value)) { - return Status::OK(); - } - switch (opt_info.verification) { - case OptionVerificationType::kByName: - case OptionVerificationType::kByNameAllowNull: - case OptionVerificationType::kByNameAllowFromNull: - return Status::NotSupported( - "Deserializing the specified CF option " + name + - " is not supported"); - case OptionVerificationType::kDeprecated: - return Status::OK(); - default: - return Status::InvalidArgument( - "Unable to parse the specified CF option " + name); - } - } - } catch (const std::exception&) { - return Status::InvalidArgument( - "unable to parse the specified option " + name); - } - return Status::OK(); -} - -template -bool SerializeSingleStructOption( - std::string* opt_string, const T& options, - const std::unordered_map& type_info, - const std::string& name, const std::string& delimiter) { - auto iter = type_info.find(name); - if (iter == type_info.end()) { - return false; - } - auto& opt_info = iter->second; - const char* opt_address = - reinterpret_cast(&options) + opt_info.offset; - std::string value; - bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value); - if (result) { - *opt_string = name + "=" + value + delimiter; - } - return result; -} - -template Status GetStringFromStruct( - std::string* opt_string, const T& options, + std::string* opt_string, const void* const options, const std::unordered_map& type_info, const std::string& delimiter) { assert(opt_string); opt_string->clear(); - for (auto iter = type_info.begin(); iter != type_info.end(); ++iter) { - if (iter->second.verification == OptionVerificationType::kDeprecated) { + for (const auto iter : type_info) { + const auto& opt_info = iter.second; + if (opt_info.IsDeprecated()) { // If the option is no longer used in rocksdb and marked as deprecated, // we skip it in the serialization. continue; } - std::string single_output; - bool result = SerializeSingleStructOption( - &single_output, options, type_info, iter->first, delimiter); + const char* opt_address = + reinterpret_cast(options) + opt_info.offset; + std::string value; + bool result = + SerializeSingleOptionHelper(opt_address, opt_info.type, &value); if (result) { - opt_string->append(single_output); + opt_string->append(iter.first + "=" + value + delimiter); } else { return Status::InvalidArgument("failed to serialize %s\n", - iter->first.c_str()); + iter.first.c_str()); } - assert(result); } return Status::OK(); } @@ -1186,15 +1036,15 @@ Status GetStringFromStruct( Status GetStringFromDBOptions(std::string* opt_string, const DBOptions& db_options, const std::string& delimiter) { - return GetStringFromStruct(opt_string, db_options, - db_options_type_info, delimiter); + return GetStringFromStruct(opt_string, &db_options, db_options_type_info, + delimiter); } Status GetStringFromColumnFamilyOptions(std::string* opt_string, const ColumnFamilyOptions& cf_options, const std::string& delimiter) { - return GetStringFromStruct( - opt_string, cf_options, cf_options_type_info, delimiter); + return GetStringFromStruct(opt_string, &cf_options, cf_options_type_info, + delimiter); } Status GetStringFromCompressionType(std::string* compression_str, @@ -1243,23 +1093,17 @@ Status ParseDBOption(const std::string& name, return Status::InvalidArgument("Unrecognized option DBOptions:", name); } const auto& opt_info = iter->second; - if (opt_info.verification != OptionVerificationType::kDeprecated && + if (opt_info.IsDeprecated() || ParseOptionHelper( reinterpret_cast(new_options) + opt_info.offset, opt_info.type, value)) { return Status::OK(); - } - switch (opt_info.verification) { - case OptionVerificationType::kByName: - case OptionVerificationType::kByNameAllowNull: - return Status::NotSupported( - "Deserializing the specified DB option " + name + - " is not supported"); - case OptionVerificationType::kDeprecated: - return Status::OK(); - default: - return Status::InvalidArgument( - "Unable to parse the specified DB option " + name); + } else if (opt_info.IsByName()) { + return Status::NotSupported("Deserializing the specified DB option " + + name + " is not supported"); + } else { + return Status::InvalidArgument( + "Unable to parse the specified DB option " + name); } } } catch (const std::exception&) { @@ -1442,285 +1286,6 @@ Status GetTableFactoryFromMap( return Status::OK(); } -std::unordered_map - OptionsHelper::db_options_type_info = { - /* - // not yet supported - std::shared_ptr row_cache; - std::shared_ptr delete_scheduler; - std::shared_ptr info_log; - std::shared_ptr rate_limiter; - std::shared_ptr statistics; - std::vector db_paths; - std::vector> listeners; - */ - {"advise_random_on_open", - {offsetof(struct DBOptions, advise_random_on_open), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"allow_mmap_reads", - {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"allow_fallocate", - {offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"allow_mmap_writes", - {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"use_direct_reads", - {offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"use_direct_writes", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"use_direct_io_for_flush_and_compaction", - {offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"allow_2pc", - {offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"allow_os_buffer", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, - 0}}, - {"create_if_missing", - {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"create_missing_column_families", - {offsetof(struct DBOptions, create_missing_column_families), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"disableDataSync", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"disable_data_sync", // for compatibility - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"enable_thread_tracking", - {offsetof(struct DBOptions, enable_thread_tracking), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"error_if_exists", - {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"is_fd_close_on_exec", - {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"paranoid_checks", - {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"skip_log_error_on_recovery", - {offsetof(struct DBOptions, skip_log_error_on_recovery), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"skip_stats_update_on_db_open", - {offsetof(struct DBOptions, skip_stats_update_on_db_open), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"skip_checking_sst_file_sizes_on_db_open", - {offsetof(struct DBOptions, skip_checking_sst_file_sizes_on_db_open), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"new_table_reader_for_compaction_inputs", - {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"compaction_readahead_size", - {offsetof(struct DBOptions, compaction_readahead_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, compaction_readahead_size)}}, - {"random_access_max_buffer_size", - {offsetof(struct DBOptions, random_access_max_buffer_size), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"use_adaptive_mutex", - {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"use_fsync", - {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"max_background_jobs", - {offsetof(struct DBOptions, max_background_jobs), OptionType::kInt, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, max_background_jobs)}}, - {"max_background_compactions", - {offsetof(struct DBOptions, max_background_compactions), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, max_background_compactions)}}, - {"base_background_compactions", - {offsetof(struct DBOptions, base_background_compactions), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, base_background_compactions)}}, - {"max_background_flushes", - {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt, - OptionVerificationType::kNormal, false, 0}}, - {"max_file_opening_threads", - {offsetof(struct DBOptions, max_file_opening_threads), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"max_open_files", - {offsetof(struct DBOptions, max_open_files), OptionType::kInt, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, max_open_files)}}, - {"table_cache_numshardbits", - {offsetof(struct DBOptions, table_cache_numshardbits), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"db_write_buffer_size", - {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"keep_log_file_num", - {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"recycle_log_file_num", - {offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"log_file_time_to_roll", - {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"manifest_preallocation_size", - {offsetof(struct DBOptions, manifest_preallocation_size), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"max_log_file_size", - {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"db_log_dir", - {offsetof(struct DBOptions, db_log_dir), OptionType::kString, - OptionVerificationType::kNormal, false, 0}}, - {"wal_dir", - {offsetof(struct DBOptions, wal_dir), OptionType::kString, - OptionVerificationType::kNormal, false, 0}}, - {"max_subcompactions", - {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T, - OptionVerificationType::kNormal, false, 0}}, - {"WAL_size_limit_MB", - {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T, - OptionVerificationType::kNormal, false, 0}}, - {"WAL_ttl_seconds", - {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T, - OptionVerificationType::kNormal, false, 0}}, - {"bytes_per_sync", - {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, bytes_per_sync)}}, - {"delayed_write_rate", - {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, delayed_write_rate)}}, - {"delete_obsolete_files_period_micros", - {offsetof(struct DBOptions, delete_obsolete_files_period_micros), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, - delete_obsolete_files_period_micros)}}, - {"max_manifest_file_size", - {offsetof(struct DBOptions, max_manifest_file_size), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"max_total_wal_size", - {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, max_total_wal_size)}}, - {"wal_bytes_per_sync", - {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, wal_bytes_per_sync)}}, - {"strict_bytes_per_sync", - {offsetof(struct DBOptions, strict_bytes_per_sync), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, strict_bytes_per_sync)}}, - {"stats_dump_period_sec", - {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, stats_dump_period_sec)}}, - {"stats_persist_period_sec", - {offsetof(struct DBOptions, stats_persist_period_sec), - OptionType::kUInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, stats_persist_period_sec)}}, - {"persist_stats_to_disk", - {offsetof(struct DBOptions, persist_stats_to_disk), - OptionType::kBoolean, OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}}, - {"stats_history_buffer_size", - {offsetof(struct DBOptions, stats_history_buffer_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, stats_history_buffer_size)}}, - {"fail_if_options_file_error", - {offsetof(struct DBOptions, fail_if_options_file_error), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"enable_pipelined_write", - {offsetof(struct DBOptions, enable_pipelined_write), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"unordered_write", - {offsetof(struct DBOptions, unordered_write), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"allow_concurrent_memtable_write", - {offsetof(struct DBOptions, allow_concurrent_memtable_write), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"wal_recovery_mode", - {offsetof(struct DBOptions, wal_recovery_mode), - OptionType::kWALRecoveryMode, OptionVerificationType::kNormal, false, - 0}}, - {"enable_write_thread_adaptive_yield", - {offsetof(struct DBOptions, enable_write_thread_adaptive_yield), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"write_thread_slow_yield_usec", - {offsetof(struct DBOptions, write_thread_slow_yield_usec), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"max_write_batch_group_size_bytes", - {offsetof(struct DBOptions, max_write_batch_group_size_bytes), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"write_thread_max_yield_usec", - {offsetof(struct DBOptions, write_thread_max_yield_usec), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"access_hint_on_compaction_start", - {offsetof(struct DBOptions, access_hint_on_compaction_start), - OptionType::kAccessHint, OptionVerificationType::kNormal, false, 0}}, - {"info_log_level", - {offsetof(struct DBOptions, info_log_level), OptionType::kInfoLogLevel, - OptionVerificationType::kNormal, false, 0}}, - {"dump_malloc_stats", - {offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"avoid_flush_during_recovery", - {offsetof(struct DBOptions, avoid_flush_during_recovery), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"avoid_flush_during_shutdown", - {offsetof(struct DBOptions, avoid_flush_during_shutdown), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}}, - {"writable_file_max_buffer_size", - {offsetof(struct DBOptions, writable_file_max_buffer_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}}, - {"allow_ingest_behind", - {offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, allow_ingest_behind)}}, - {"preserve_deletes", - {offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, preserve_deletes)}}, - {"concurrent_prepare", // Deprecated by two_write_queues - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"two_write_queues", - {offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, two_write_queues)}}, - {"manual_wal_flush", - {offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, manual_wal_flush)}}, - {"seq_per_batch", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"atomic_flush", - {offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, atomic_flush)}}, - {"avoid_unnecessary_blocking_io", - {offsetof(struct DBOptions, avoid_unnecessary_blocking_io), - OptionType::kBoolean, OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}}, - {"write_dbid_to_manifest", - {offsetof(struct DBOptions, write_dbid_to_manifest), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"log_readahead_size", - {offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"best_efforts_recovery", - {offsetof(struct DBOptions, best_efforts_recovery), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, -}; - std::unordered_map OptionsHelper::block_base_table_index_type_string_map = { {"kBinarySearch", BlockBasedTableOptions::IndexType::kBinarySearch}, @@ -1790,347 +1355,81 @@ std::unordered_map {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL}, {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}}; -ColumnFamilyOptions OptionsHelper::dummy_cf_options; -CompactionOptionsFIFO OptionsHelper::dummy_comp_options; LRUCacheOptions OptionsHelper::dummy_lru_cache_options; CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal; +CompactionOptionsFIFO OptionsHelper::dummy_comp_options; -// offset_of is used to get the offset of a class data member -// ex: offset_of(&ColumnFamilyOptions::num_levels) -// This call will return the offset of num_levels in ColumnFamilyOptions class -// -// This is the same as offsetof() but allow us to work with non standard-layout -// classes and structures -// refs: -// http://en.cppreference.com/w/cpp/concept/StandardLayoutType -// https://gist.github.com/graphitemaster/494f21190bb2c63c5516 -template -int offset_of(T1 ColumnFamilyOptions::*member) { - return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - - size_t(&OptionsHelper::dummy_cf_options)); -} template -int offset_of(T1 AdvancedColumnFamilyOptions::*member) { - return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - - size_t(&OptionsHelper::dummy_cf_options)); +int offset_of(T1 LRUCacheOptions::*member) { + return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) - + size_t(&OptionsHelper::dummy_lru_cache_options)); } + template int offset_of(T1 CompactionOptionsFIFO::*member) { return int(size_t(&(OptionsHelper::dummy_comp_options.*member)) - size_t(&OptionsHelper::dummy_comp_options)); } template -int offset_of(T1 LRUCacheOptions::*member) { - return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) - - size_t(&OptionsHelper::dummy_lru_cache_options)); -} -template int offset_of(T1 CompactionOptionsUniversal::*member) { return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) - size_t(&OptionsHelper::dummy_comp_options_universal)); } -std::unordered_map - OptionsHelper::cf_options_type_info = { - /* not yet supported - CompressionOptions compression_opts; - TablePropertiesCollectorFactories table_properties_collector_factories; - typedef std::vector> - TablePropertiesCollectorFactories; - UpdateStatus (*inplace_callback)(char* existing_value, - uint34_t* existing_value_size, - Slice delta_value, - std::string* merged_value); - std::vector cf_paths; - */ - {"report_bg_io_stats", - {offset_of(&ColumnFamilyOptions::report_bg_io_stats), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, report_bg_io_stats)}}, - {"compaction_measure_io_stats", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"disable_auto_compactions", - {offset_of(&ColumnFamilyOptions::disable_auto_compactions), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, disable_auto_compactions)}}, - {"filter_deletes", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, - 0}}, - {"inplace_update_support", - {offset_of(&ColumnFamilyOptions::inplace_update_support), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"level_compaction_dynamic_level_bytes", - {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"optimize_filters_for_hits", - {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"paranoid_file_checks", - {offset_of(&ColumnFamilyOptions::paranoid_file_checks), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, paranoid_file_checks)}}, - {"force_consistency_checks", - {offset_of(&ColumnFamilyOptions::force_consistency_checks), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"purge_redundant_kvs_while_flush", - {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush), - OptionType::kBoolean, OptionVerificationType::kDeprecated, false, 0}}, - {"verify_checksums_in_compaction", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, - 0}}, - {"soft_pending_compaction_bytes_limit", - {offset_of(&ColumnFamilyOptions::soft_pending_compaction_bytes_limit), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - soft_pending_compaction_bytes_limit)}}, - {"hard_pending_compaction_bytes_limit", - {offset_of(&ColumnFamilyOptions::hard_pending_compaction_bytes_limit), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - hard_pending_compaction_bytes_limit)}}, - {"hard_rate_limit", - {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true, - 0}}, - {"soft_rate_limit", - {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true, - 0}}, - {"max_compaction_bytes", - {offset_of(&ColumnFamilyOptions::max_compaction_bytes), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_compaction_bytes)}}, - {"expanded_compaction_factor", - {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}}, - {"level0_file_num_compaction_trigger", - {offset_of(&ColumnFamilyOptions::level0_file_num_compaction_trigger), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - level0_file_num_compaction_trigger)}}, - {"level0_slowdown_writes_trigger", - {offset_of(&ColumnFamilyOptions::level0_slowdown_writes_trigger), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger)}}, - {"level0_stop_writes_trigger", - {offset_of(&ColumnFamilyOptions::level0_stop_writes_trigger), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, level0_stop_writes_trigger)}}, - {"max_grandparent_overlap_factor", - {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}}, - {"max_mem_compaction_level", - {0, OptionType::kInt, OptionVerificationType::kDeprecated, false, 0}}, - {"max_write_buffer_number", - {offset_of(&ColumnFamilyOptions::max_write_buffer_number), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_write_buffer_number)}}, - {"max_write_buffer_number_to_maintain", - {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"max_write_buffer_size_to_maintain", - {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain), - OptionType::kInt64T, OptionVerificationType::kNormal, false, 0}}, - {"min_write_buffer_number_to_merge", - {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"num_levels", - {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt, - OptionVerificationType::kNormal, false, 0}}, - {"source_compaction_factor", - {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}}, - {"target_file_size_multiplier", - {offset_of(&ColumnFamilyOptions::target_file_size_multiplier), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, target_file_size_multiplier)}}, - {"arena_block_size", - {offset_of(&ColumnFamilyOptions::arena_block_size), OptionType::kSizeT, - OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, arena_block_size)}}, - {"inplace_update_num_locks", - {offset_of(&ColumnFamilyOptions::inplace_update_num_locks), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, inplace_update_num_locks)}}, - {"max_successive_merges", - {offset_of(&ColumnFamilyOptions::max_successive_merges), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_successive_merges)}}, - {"memtable_huge_page_size", - {offset_of(&ColumnFamilyOptions::memtable_huge_page_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, memtable_huge_page_size)}}, - {"memtable_prefix_bloom_huge_page_tlb_size", - {0, OptionType::kSizeT, OptionVerificationType::kDeprecated, true, 0}}, - {"write_buffer_size", - {offset_of(&ColumnFamilyOptions::write_buffer_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, write_buffer_size)}}, - {"bloom_locality", - {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T, - OptionVerificationType::kNormal, false, 0}}, - {"memtable_prefix_bloom_bits", - {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, - 0}}, - {"memtable_prefix_bloom_size_ratio", - {offset_of(&ColumnFamilyOptions::memtable_prefix_bloom_size_ratio), - OptionType::kDouble, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio)}}, - {"memtable_prefix_bloom_probes", - {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, - 0}}, - {"memtable_whole_key_filtering", - {offset_of(&ColumnFamilyOptions::memtable_whole_key_filtering), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, memtable_whole_key_filtering)}}, - {"min_partial_merge_operands", - {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, - 0}}, - {"max_bytes_for_level_base", - {offset_of(&ColumnFamilyOptions::max_bytes_for_level_base), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_bytes_for_level_base)}}, - {"snap_refresh_nanos", - {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, true, - 0}}, - {"max_bytes_for_level_multiplier", - {offset_of(&ColumnFamilyOptions::max_bytes_for_level_multiplier), - OptionType::kDouble, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}}, - {"max_bytes_for_level_multiplier_additional", - {offset_of( - &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional), - OptionType::kVectorInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - max_bytes_for_level_multiplier_additional)}}, - {"max_sequential_skip_in_iterations", - {offset_of(&ColumnFamilyOptions::max_sequential_skip_in_iterations), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - max_sequential_skip_in_iterations)}}, - {"target_file_size_base", - {offset_of(&ColumnFamilyOptions::target_file_size_base), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, target_file_size_base)}}, - {"rate_limit_delay_max_milliseconds", - {0, OptionType::kUInt, OptionVerificationType::kDeprecated, false, 0}}, - {"compression", - {offset_of(&ColumnFamilyOptions::compression), - OptionType::kCompressionType, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, compression)}}, - {"compression_per_level", - {offset_of(&ColumnFamilyOptions::compression_per_level), - OptionType::kVectorCompressionType, OptionVerificationType::kNormal, - false, 0}}, - {"bottommost_compression", - {offset_of(&ColumnFamilyOptions::bottommost_compression), - OptionType::kCompressionType, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, bottommost_compression)}}, - {kNameComparator, - {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator, - OptionVerificationType::kByName, false, 0}}, - {"prefix_extractor", - {offset_of(&ColumnFamilyOptions::prefix_extractor), - OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, - true, offsetof(struct MutableCFOptions, prefix_extractor)}}, - {"memtable_insert_with_hint_prefix_extractor", - {offset_of( - &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), - OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, - false, 0}}, - {"memtable_factory", - {offset_of(&ColumnFamilyOptions::memtable_factory), - OptionType::kMemTableRepFactory, OptionVerificationType::kByName, - false, 0}}, - {"table_factory", - {offset_of(&ColumnFamilyOptions::table_factory), - OptionType::kTableFactory, OptionVerificationType::kByName, false, - 0}}, - {"compaction_filter", - {offset_of(&ColumnFamilyOptions::compaction_filter), - OptionType::kCompactionFilter, OptionVerificationType::kByName, false, - 0}}, - {"compaction_filter_factory", - {offset_of(&ColumnFamilyOptions::compaction_filter_factory), - OptionType::kCompactionFilterFactory, OptionVerificationType::kByName, - false, 0}}, - {kNameMergeOperator, - {offset_of(&ColumnFamilyOptions::merge_operator), - OptionType::kMergeOperator, - OptionVerificationType::kByNameAllowFromNull, false, 0}}, - {"compaction_style", - {offset_of(&ColumnFamilyOptions::compaction_style), - OptionType::kCompactionStyle, OptionVerificationType::kNormal, false, - 0}}, - {"compaction_pri", - {offset_of(&ColumnFamilyOptions::compaction_pri), - OptionType::kCompactionPri, OptionVerificationType::kNormal, false, - 0}}, - {"compaction_options_fifo", - {offset_of(&ColumnFamilyOptions::compaction_options_fifo), - OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal, - true, offsetof(struct MutableCFOptions, compaction_options_fifo)}}, - {"compaction_options_universal", - {offset_of(&ColumnFamilyOptions::compaction_options_universal), - OptionType::kCompactionOptionsUniversal, - OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, compaction_options_universal)}}, - {"ttl", - {offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, ttl)}}, - {"periodic_compaction_seconds", - {offset_of(&ColumnFamilyOptions::periodic_compaction_seconds), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, periodic_compaction_seconds)}}, - {"sample_for_compression", - {offset_of(&ColumnFamilyOptions::sample_for_compression), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, sample_for_compression)}}}; - std::unordered_map OptionsHelper::fifo_compaction_options_type_info = { {"max_table_files_size", {offset_of(&CompactionOptionsFIFO::max_table_files_size), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(struct CompactionOptionsFIFO, max_table_files_size)}}, {"ttl", - {0, OptionType::kUInt64T, - OptionVerificationType::kDeprecated, false, - 0}}, + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, {"allow_compaction", {offset_of(&CompactionOptionsFIFO::allow_compaction), - OptionType::kBoolean, OptionVerificationType::kNormal, true, + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(struct CompactionOptionsFIFO, allow_compaction)}}}; std::unordered_map OptionsHelper::universal_compaction_options_type_info = { {"size_ratio", {offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt, - OptionVerificationType::kNormal, true, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, size_ratio)}}, {"min_merge_width", {offset_of(&CompactionOptionsUniversal::min_merge_width), - OptionType::kUInt, OptionVerificationType::kNormal, true, + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, min_merge_width)}}, {"max_merge_width", {offset_of(&CompactionOptionsUniversal::max_merge_width), - OptionType::kUInt, OptionVerificationType::kNormal, true, + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, max_merge_width)}}, {"max_size_amplification_percent", {offset_of( &CompactionOptionsUniversal::max_size_amplification_percent), - OptionType::kUInt, OptionVerificationType::kNormal, true, + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, max_size_amplification_percent)}}, {"compression_size_percent", {offset_of(&CompactionOptionsUniversal::compression_size_percent), - OptionType::kInt, OptionVerificationType::kNormal, true, + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, compression_size_percent)}}, {"stop_style", {offset_of(&CompactionOptionsUniversal::stop_style), OptionType::kCompactionStopStyle, OptionVerificationType::kNormal, - true, offsetof(class CompactionOptionsUniversal, stop_style)}}, + OptionTypeFlags::kMutable, + offsetof(class CompactionOptionsUniversal, stop_style)}}, {"allow_trivial_move", {offset_of(&CompactionOptionsUniversal::allow_trivial_move), - OptionType::kBoolean, OptionVerificationType::kNormal, true, + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}}; std::unordered_map @@ -2142,19 +1441,20 @@ std::unordered_map OptionsHelper::lru_cache_options_type_info = { {"capacity", {offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT, - OptionVerificationType::kNormal, true, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct LRUCacheOptions, capacity)}}, {"num_shard_bits", {offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt, - OptionVerificationType::kNormal, true, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct LRUCacheOptions, num_shard_bits)}}, {"strict_capacity_limit", {offset_of(&LRUCacheOptions::strict_capacity_limit), - OptionType::kBoolean, OptionVerificationType::kNormal, true, + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(struct LRUCacheOptions, strict_capacity_limit)}}, {"high_pri_pool_ratio", {offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble, - OptionVerificationType::kNormal, true, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; #endif // !ROCKSDB_LITE diff --git a/options/options_helper.h b/options/options_helper.h index c75fdb1ce..6feca8f3d 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -12,6 +12,7 @@ #include "options/cf_options.h" #include "options/db_options.h" +#include "options/options_type.h" #include "rocksdb/options.h" #include "rocksdb/status.h" #include "rocksdb/table.h" @@ -28,6 +29,11 @@ ColumnFamilyOptions BuildColumnFamilyOptions( #ifndef ROCKSDB_LITE +Status ParseColumnFamilyOption(const std::string& name, + const std::string& org_value, + ColumnFamilyOptions* new_options, + bool input_strings_escaped = false); + Status GetMutableOptionsFromStrings( const MutableCFOptions& base_options, const std::unordered_map& options_map, @@ -44,71 +50,9 @@ Status GetTableFactoryFromMap( std::shared_ptr* table_factory, bool ignore_unknown_options = false); -enum class OptionType { - kBoolean, - kInt, - kInt32T, - kInt64T, - kVectorInt, - kUInt, - kUInt32T, - kUInt64T, - kSizeT, - kString, - kDouble, - kCompactionStyle, - kCompactionPri, - kSliceTransform, - kCompressionType, - kVectorCompressionType, - kTableFactory, - kComparator, - kCompactionFilter, - kCompactionFilterFactory, - kCompactionOptionsFIFO, - kCompactionOptionsUniversal, - kCompactionStopStyle, - kMergeOperator, - kMemTableRepFactory, - kBlockBasedTableIndexType, - kBlockBasedTableDataBlockIndexType, - kBlockBasedTableIndexShorteningMode, - kFilterPolicy, - kFlushBlockPolicyFactory, - kChecksumType, - kEncodingType, - kWALRecoveryMode, - kAccessHint, - kInfoLogLevel, - kLRUCacheOptions, - kEnv, - kUnknown, -}; - -enum class OptionVerificationType { - kNormal, - kByName, // The option is pointer typed so we can only verify - // based on it's name. - kByNameAllowNull, // Same as kByName, but it also allows the case - // where one of them is a nullptr. - kByNameAllowFromNull, // Same as kByName, but it also allows the case - // where the old option is nullptr. - kDeprecated // The option is no longer used in rocksdb. The RocksDB - // OptionsParser will still accept this option if it - // happen to exists in some Options file. However, - // the parser will not include it in serialization - // and verification processes. -}; - -// A struct for storing constant option information such as option name, -// option type, and offset. -struct OptionTypeInfo { - int offset; - OptionType type; - OptionVerificationType verification; - bool is_mutable; - int mutable_offset; -}; +Status ParseCompressionOptions(const std::string& value, + const std::string& name, + CompressionOptions& compression_opts); // A helper function that converts "opt_address" to a std::string // based on the specified OptionType. @@ -145,6 +89,10 @@ extern Status StringToMap( extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, const std::string& value); +Status GetStringFromStruct( + std::string* opt_string, const void* const options, + const std::unordered_map& type_info, + const std::string& delimiter); #endif // !ROCKSDB_LITE struct OptionsHelper { diff --git a/options/options_parser.cc b/options/options_parser.cc index 12be6a870..0f8eecc9a 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -620,10 +620,7 @@ bool AreEqualOptions( return false; } default: - if (type_info.verification == OptionVerificationType::kByName || - type_info.verification == - OptionVerificationType::kByNameAllowFromNull || - type_info.verification == OptionVerificationType::kByNameAllowNull) { + if (type_info.IsByName()) { std::string value1; bool result = SerializeSingleOptionHelper(offset1, type_info.type, &value1); @@ -637,13 +634,12 @@ bool AreEqualOptions( if (iter == opt_map->end()) { return true; } else { - if (type_info.verification == - OptionVerificationType::kByNameAllowNull) { + if (type_info.IsEnabled(OptionVerificationType::kByNameAllowNull)) { if (iter->second == kNullptrString || value1 == kNullptrString) { return true; } - } else if (type_info.verification == - OptionVerificationType::kByNameAllowFromNull) { + } else if (type_info.IsEnabled( + OptionVerificationType::kByNameAllowFromNull)) { if (iter->second == kNullptrString) { return true; } @@ -684,7 +680,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( // Verify ColumnFamily Name if (cf_names.size() != parser.cf_names()->size()) { - if (sanity_check_level >= kSanityLevelLooselyCompatible) { + if (sanity_check_level >= + OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) { return Status::InvalidArgument( "[RocksDBOptionParser Error] The persisted options does not have " "the same number of column family names as the db instance."); @@ -706,7 +703,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( // Verify Column Family Options if (cf_opts.size() != parser.cf_opts()->size()) { - if (sanity_check_level >= kSanityLevelLooselyCompatible) { + if (sanity_check_level >= + OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) { return Status::InvalidArgument( "[RocksDBOptionsParser Error]", "The persisted options does not have the same number of " @@ -740,7 +738,7 @@ Status RocksDBOptionsParser::VerifyDBOptions( const std::unordered_map* /*opt_map*/, OptionsSanityCheckLevel sanity_check_level) { for (const auto& pair : db_options_type_info) { - if (pair.second.verification == OptionVerificationType::kDeprecated) { + if (pair.second.IsDeprecated()) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; @@ -778,7 +776,7 @@ Status RocksDBOptionsParser::VerifyCFOptions( const std::unordered_map* persisted_opt_map, OptionsSanityCheckLevel sanity_check_level) { for (const auto& pair : cf_options_type_info) { - if (pair.second.verification == OptionVerificationType::kDeprecated) { + if (pair.second.IsDeprecated()) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; @@ -814,7 +812,7 @@ Status RocksDBOptionsParser::VerifyTableFactory( const TableFactory* base_tf, const TableFactory* file_tf, OptionsSanityCheckLevel sanity_check_level) { if (base_tf && file_tf) { - if (sanity_check_level > kSanityLevelNone && + if (sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone && std::string(base_tf->Name()) != std::string(file_tf->Name())) { return Status::Corruption( "[RocksDBOptionsParser]: " diff --git a/options/options_type.h b/options/options_type.h new file mode 100644 index 000000000..51e0c0031 --- /dev/null +++ b/options/options_type.h @@ -0,0 +1,141 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +enum class OptionType { + kBoolean, + kInt, + kInt32T, + kInt64T, + kVectorInt, + kUInt, + kUInt32T, + kUInt64T, + kSizeT, + kString, + kDouble, + kCompactionStyle, + kCompactionPri, + kSliceTransform, + kCompressionType, + kVectorCompressionType, + kTableFactory, + kComparator, + kCompactionFilter, + kCompactionFilterFactory, + kCompactionOptionsFIFO, + kCompactionOptionsUniversal, + kCompactionStopStyle, + kMergeOperator, + kMemTableRepFactory, + kBlockBasedTableIndexType, + kBlockBasedTableDataBlockIndexType, + kBlockBasedTableIndexShorteningMode, + kFilterPolicy, + kFlushBlockPolicyFactory, + kChecksumType, + kEncodingType, + kWALRecoveryMode, + kAccessHint, + kInfoLogLevel, + kLRUCacheOptions, + kEnv, + kUnknown, +}; + +enum class OptionVerificationType { + kNormal, + kByName, // The option is pointer typed so we can only verify + // based on it's name. + kByNameAllowNull, // Same as kByName, but it also allows the case + // where one of them is a nullptr. + kByNameAllowFromNull, // Same as kByName, but it also allows the case + // where the old option is nullptr. + kDeprecated // The option is no longer used in rocksdb. The RocksDB + // OptionsParser will still accept this option if it + // happen to exists in some Options file. However, + // the parser will not include it in serialization + // and verification processes. +}; + +enum class OptionTypeFlags : uint32_t { + kNone = 0x00, // No flags + kMutable = 0x01, // Option is mutable +}; + +inline OptionTypeFlags operator|(const OptionTypeFlags &a, + const OptionTypeFlags &b) { + return static_cast(static_cast(a) | + static_cast(b)); +} + +inline OptionTypeFlags operator&(const OptionTypeFlags &a, + const OptionTypeFlags &b) { + return static_cast(static_cast(a) & + static_cast(b)); +} + +// A struct for storing constant option information such as option name, +// option type, and offset. +class OptionTypeInfo { + public: + int offset; + int mutable_offset; + OptionType type; + + // A simple "normal", non-mutable Type "_type" at _offset + OptionTypeInfo(int _offset, OptionType _type) + : offset(_offset), + mutable_offset(0), + type(_type), + verification(OptionVerificationType::kNormal), + flags(OptionTypeFlags::kNone) {} + + // A simple "normal", mutable Type "_type" at _offset + OptionTypeInfo(int _offset, OptionType _type, int _mutable_offset) + : offset(_offset), + mutable_offset(_mutable_offset), + type(_type), + verification(OptionVerificationType::kNormal), + flags(OptionTypeFlags::kMutable) {} + + OptionTypeInfo(int _offset, OptionType _type, + OptionVerificationType _verification, OptionTypeFlags _flags, + int _mutable_offset) + : offset(_offset), + mutable_offset(_mutable_offset), + type(_type), + verification(_verification), + flags(_flags) {} + + bool IsEnabled(OptionTypeFlags otf) const { return (flags & otf) == otf; } + + bool IsMutable() const { return IsEnabled(OptionTypeFlags::kMutable); } + + bool IsDeprecated() const { + return IsEnabled(OptionVerificationType::kDeprecated); + } + + bool IsEnabled(OptionVerificationType ovf) const { + return verification == ovf; + } + + bool IsByName() const { + return (verification == OptionVerificationType::kByName || + verification == OptionVerificationType::kByNameAllowNull || + verification == OptionVerificationType::kByNameAllowFromNull); + } + + protected: + private: + OptionVerificationType verification; + OptionTypeFlags flags; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index 70a6f38d5..b79f8e7de 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -157,6 +157,126 @@ size_t TailPrefetchStats::GetSuggestedPrefetchSize() { return std::min(kMaxPrefetchSize, max_qualified_size); } +#ifndef ROCKSDB_LITE +static std::unordered_map + block_based_table_type_info = { + /* currently not supported + std::shared_ptr block_cache = nullptr; + std::shared_ptr block_cache_compressed = nullptr; + */ + {"flush_block_policy_factory", + {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), + OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"cache_index_and_filter_blocks", + {offsetof(struct BlockBasedTableOptions, + cache_index_and_filter_blocks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"cache_index_and_filter_blocks_with_high_priority", + {offsetof(struct BlockBasedTableOptions, + cache_index_and_filter_blocks_with_high_priority), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"pin_l0_filter_and_index_blocks_in_cache", + {offsetof(struct BlockBasedTableOptions, + pin_l0_filter_and_index_blocks_in_cache), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"index_type", + {offsetof(struct BlockBasedTableOptions, index_type), + OptionType::kBlockBasedTableIndexType, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"hash_index_allow_collision", + {offsetof(struct BlockBasedTableOptions, hash_index_allow_collision), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"data_block_index_type", + {offsetof(struct BlockBasedTableOptions, data_block_index_type), + OptionType::kBlockBasedTableDataBlockIndexType, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"index_shortening", + {offsetof(struct BlockBasedTableOptions, index_shortening), + OptionType::kBlockBasedTableIndexShorteningMode, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"data_block_hash_table_util_ratio", + {offsetof(struct BlockBasedTableOptions, + data_block_hash_table_util_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"checksum", + {offsetof(struct BlockBasedTableOptions, checksum), + OptionType::kChecksumType, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"no_block_cache", + {offsetof(struct BlockBasedTableOptions, no_block_cache), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"block_size", + {offsetof(struct BlockBasedTableOptions, block_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"block_size_deviation", + {offsetof(struct BlockBasedTableOptions, block_size_deviation), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"block_restart_interval", + {offsetof(struct BlockBasedTableOptions, block_restart_interval), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"index_block_restart_interval", + {offsetof(struct BlockBasedTableOptions, index_block_restart_interval), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"index_per_partition", + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"metadata_block_size", + {offsetof(struct BlockBasedTableOptions, metadata_block_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"partition_filters", + {offsetof(struct BlockBasedTableOptions, partition_filters), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"filter_policy", + {offsetof(struct BlockBasedTableOptions, filter_policy), + OptionType::kFilterPolicy, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"whole_key_filtering", + {offsetof(struct BlockBasedTableOptions, whole_key_filtering), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"skip_table_builder_flush", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"format_version", + {offsetof(struct BlockBasedTableOptions, format_version), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"verify_compression", + {offsetof(struct BlockBasedTableOptions, verify_compression), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"read_amp_bytes_per_bit", + {offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"enable_index_compression", + {offsetof(struct BlockBasedTableOptions, enable_index_compression), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"block_align", + {offsetof(struct BlockBasedTableOptions, block_align), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"pin_top_level_index_and_filter", + {offsetof(struct BlockBasedTableOptions, + pin_top_level_index_and_filter), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}}; +#endif // ROCKSDB_LITE + // TODO(myabandeh): We should return an error instead of silently changing the // options BlockBasedTableFactory::BlockBasedTableFactory( @@ -416,46 +536,13 @@ std::string BlockBasedTableFactory::GetPrintableTableOptions() const { } #ifndef ROCKSDB_LITE -namespace { -bool SerializeSingleBlockBasedTableOption( - std::string* opt_string, const BlockBasedTableOptions& bbt_options, - const std::string& name, const std::string& delimiter) { - auto iter = block_based_table_type_info.find(name); - if (iter == block_based_table_type_info.end()) { - return false; - } - auto& opt_info = iter->second; - const char* opt_address = - reinterpret_cast(&bbt_options) + opt_info.offset; - std::string value; - bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value); - if (result) { - *opt_string = name + "=" + value + delimiter; - } - return result; -} -} // namespace Status BlockBasedTableFactory::GetOptionString( std::string* opt_string, const std::string& delimiter) const { assert(opt_string); opt_string->clear(); - for (auto iter = block_based_table_type_info.begin(); - iter != block_based_table_type_info.end(); ++iter) { - if (iter->second.verification == OptionVerificationType::kDeprecated) { - // If the option is no longer used in rocksdb and marked as deprecated, - // we skip it in the serialization. - continue; - } - std::string single_output; - bool result = SerializeSingleBlockBasedTableOption( - &single_output, table_options_, iter->first, delimiter); - assert(result); - if (result) { - opt_string->append(single_output); - } - } - return Status::OK(); + return GetStringFromStruct(opt_string, &table_options_, + block_based_table_type_info, delimiter); } #else Status BlockBasedTableFactory::GetOptionString( @@ -535,7 +622,7 @@ std::string ParseBlockBasedTableOption(const std::string& name, } } const auto& opt_info = iter->second; - if (opt_info.verification != OptionVerificationType::kDeprecated && + if (!opt_info.IsDeprecated() && !ParseOptionHelper(reinterpret_cast(new_options) + opt_info.offset, opt_info.type, value)) { return "Invalid value"; @@ -574,12 +661,7 @@ Status GetBlockBasedTableOptionsFromMap( !input_strings_escaped || // !input_strings_escaped indicates // the old API, where everything is // parsable. - (iter->second.verification != OptionVerificationType::kByName && - iter->second.verification != - OptionVerificationType::kByNameAllowNull && - iter->second.verification != - OptionVerificationType::kByNameAllowFromNull && - iter->second.verification != OptionVerificationType::kDeprecated)) { + (!iter->second.IsByName() && !iter->second.IsDeprecated())) { // Restore "new_options" to the default "base_options". *new_table_options = table_options; return Status::InvalidArgument("Can't parse BlockBasedTableOptions:", @@ -595,7 +677,7 @@ Status VerifyBlockBasedTableFactory( const BlockBasedTableFactory* file_tf, OptionsSanityCheckLevel sanity_check_level) { if ((base_tf != nullptr) != (file_tf != nullptr) && - sanity_check_level > kSanityLevelNone) { + sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone) { return Status::Corruption( "[RocksDBOptionsParser]: Inconsistent TableFactory class type"); } @@ -608,7 +690,7 @@ Status VerifyBlockBasedTableFactory( const auto& file_opt = file_tf->table_options(); for (auto& pair : block_based_table_type_info) { - if (pair.second.verification == OptionVerificationType::kDeprecated) { + if (pair.second.IsDeprecated()) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; diff --git a/table/block_based/block_based_table_factory.h b/table/block_based/block_based_table_factory.h index 7c8633c07..6bc01cee6 100644 --- a/table/block_based/block_based_table_factory.h +++ b/table/block_based/block_based_table_factory.h @@ -93,103 +93,5 @@ extern Status VerifyBlockBasedTableFactory( const BlockBasedTableFactory* file_tf, OptionsSanityCheckLevel sanity_check_level); -static std::unordered_map - block_based_table_type_info = { - /* currently not supported - std::shared_ptr block_cache = nullptr; - std::shared_ptr block_cache_compressed = nullptr; - */ - {"flush_block_policy_factory", - {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), - OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName, - false, 0}}, - {"cache_index_and_filter_blocks", - {offsetof(struct BlockBasedTableOptions, - cache_index_and_filter_blocks), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"cache_index_and_filter_blocks_with_high_priority", - {offsetof(struct BlockBasedTableOptions, - cache_index_and_filter_blocks_with_high_priority), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"pin_l0_filter_and_index_blocks_in_cache", - {offsetof(struct BlockBasedTableOptions, - pin_l0_filter_and_index_blocks_in_cache), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"index_type", - {offsetof(struct BlockBasedTableOptions, index_type), - OptionType::kBlockBasedTableIndexType, - OptionVerificationType::kNormal, false, 0}}, - {"hash_index_allow_collision", - {offsetof(struct BlockBasedTableOptions, hash_index_allow_collision), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"data_block_index_type", - {offsetof(struct BlockBasedTableOptions, data_block_index_type), - OptionType::kBlockBasedTableDataBlockIndexType, - OptionVerificationType::kNormal, false, 0}}, - {"index_shortening", - {offsetof(struct BlockBasedTableOptions, index_shortening), - OptionType::kBlockBasedTableIndexShorteningMode, - OptionVerificationType::kNormal, false, 0}}, - {"data_block_hash_table_util_ratio", - {offsetof(struct BlockBasedTableOptions, - data_block_hash_table_util_ratio), - OptionType::kDouble, OptionVerificationType::kNormal, false, 0}}, - {"checksum", - {offsetof(struct BlockBasedTableOptions, checksum), - OptionType::kChecksumType, OptionVerificationType::kNormal, false, - 0}}, - {"no_block_cache", - {offsetof(struct BlockBasedTableOptions, no_block_cache), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"block_size", - {offsetof(struct BlockBasedTableOptions, block_size), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"block_size_deviation", - {offsetof(struct BlockBasedTableOptions, block_size_deviation), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"block_restart_interval", - {offsetof(struct BlockBasedTableOptions, block_restart_interval), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"index_block_restart_interval", - {offsetof(struct BlockBasedTableOptions, index_block_restart_interval), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"index_per_partition", - {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, false, - 0}}, - {"metadata_block_size", - {offsetof(struct BlockBasedTableOptions, metadata_block_size), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"partition_filters", - {offsetof(struct BlockBasedTableOptions, partition_filters), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"filter_policy", - {offsetof(struct BlockBasedTableOptions, filter_policy), - OptionType::kFilterPolicy, OptionVerificationType::kByName, false, - 0}}, - {"whole_key_filtering", - {offsetof(struct BlockBasedTableOptions, whole_key_filtering), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"skip_table_builder_flush", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"format_version", - {offsetof(struct BlockBasedTableOptions, format_version), - OptionType::kUInt32T, OptionVerificationType::kNormal, false, 0}}, - {"verify_compression", - {offsetof(struct BlockBasedTableOptions, verify_compression), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"read_amp_bytes_per_bit", - {offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"enable_index_compression", - {offsetof(struct BlockBasedTableOptions, enable_index_compression), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"block_align", - {offsetof(struct BlockBasedTableOptions, block_align), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"pin_top_level_index_and_filter", - {offsetof(struct BlockBasedTableOptions, - pin_top_level_index_and_filter), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}}; #endif // !ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc index c2db8f395..df296cdff 100644 --- a/table/plain/plain_table_factory.cc +++ b/table/plain/plain_table_factory.cc @@ -17,6 +17,34 @@ #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { +static std::unordered_map plain_table_type_info = { + {"user_key_len", + {offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"bloom_bits_per_key", + {offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"hash_table_ratio", + {offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"index_sparseness", + {offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"huge_page_tlb_size", + {offsetof(struct PlainTableOptions, huge_page_tlb_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"encoding_type", + {offsetof(struct PlainTableOptions, encoding_type), + OptionType::kEncodingType, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"full_scan_mode", + {offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"store_index_in_file", + {offsetof(struct PlainTableOptions, store_index_in_file), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}}; Status PlainTableFactory::NewTableReader( const TableReaderOptions& table_reader_options, @@ -178,7 +206,7 @@ std::string ParsePlainTableOptions(const std::string& name, } } const auto& opt_info = iter->second; - if (opt_info.verification != OptionVerificationType::kDeprecated && + if (!opt_info.IsDeprecated() && !ParseOptionHelper(reinterpret_cast(new_options) + opt_info.offset, opt_info.type, value)) { return "Invalid value"; @@ -202,12 +230,7 @@ Status GetPlainTableOptionsFromMap( !input_strings_escaped || // !input_strings_escaped indicates // the old API, where everything is // parsable. - (iter->second.verification != OptionVerificationType::kByName && - iter->second.verification != - OptionVerificationType::kByNameAllowNull && - iter->second.verification != - OptionVerificationType::kByNameAllowFromNull && - iter->second.verification != OptionVerificationType::kDeprecated)) { + (!iter->second.IsByName() && !iter->second.IsDeprecated())) { // Restore "new_options" to the default "base_options". *new_table_options = table_options; return Status::InvalidArgument("Can't parse PlainTableOptions:", diff --git a/table/plain/plain_table_factory.h b/table/plain/plain_table_factory.h index 64dd171cb..cbd892a9e 100644 --- a/table/plain/plain_table_factory.h +++ b/table/plain/plain_table_factory.h @@ -193,31 +193,6 @@ class PlainTableFactory : public TableFactory { PlainTableOptions table_options_; }; -static std::unordered_map plain_table_type_info = { - {"user_key_len", - {offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T, - OptionVerificationType::kNormal, false, 0}}, - {"bloom_bits_per_key", - {offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt, - OptionVerificationType::kNormal, false, 0}}, - {"hash_table_ratio", - {offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble, - OptionVerificationType::kNormal, false, 0}}, - {"index_sparseness", - {offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"huge_page_tlb_size", - {offsetof(struct PlainTableOptions, huge_page_tlb_size), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"encoding_type", - {offsetof(struct PlainTableOptions, encoding_type), - OptionType::kEncodingType, OptionVerificationType::kByName, false, 0}}, - {"full_scan_mode", - {offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"store_index_in_file", - {offsetof(struct PlainTableOptions, store_index_in_file), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}}; } // namespace ROCKSDB_NAMESPACE #endif // ROCKSDB_LITE