diff --git a/db/db_options_test.cc b/db/db_options_test.cc index 29db9d095..fc404f112 100644 --- a/db/db_options_test.cc +++ b/db/db_options_test.cc @@ -38,8 +38,7 @@ class DBOptionsTest : public DBTestBase { StringToMap(options_str, &options_map); std::unordered_map mutable_map; for (const auto opt : db_options_type_info) { - if (opt.second.is_mutable && - opt.second.verification != OptionVerificationType::kDeprecated) { + if (opt.second.IsMutable() && !opt.second.IsDeprecated()) { mutable_map[opt.first] = options_map[opt.first]; } } @@ -54,8 +53,7 @@ class DBOptionsTest : public DBTestBase { StringToMap(options_str, &options_map); std::unordered_map mutable_map; for (const auto opt : cf_options_type_info) { - if (opt.second.is_mutable && - opt.second.verification != OptionVerificationType::kDeprecated) { + if (opt.second.IsMutable() && !opt.second.IsDeprecated()) { mutable_map[opt.first] = options_map[opt.first]; } } diff --git a/options/cf_options.cc b/options/cf_options.cc index 37aa3de2c..65b198510 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -9,14 +9,451 @@ #include #include #include + #include "options/db_options.h" +#include "options/options_helper.h" #include "port/port.h" #include "rocksdb/concurrent_task_limiter.h" +#include "rocksdb/convenience.h" #include "rocksdb/env.h" #include "rocksdb/file_system.h" +#include "rocksdb/merge_operator.h" #include "rocksdb/options.h" +#include "rocksdb/utilities/object_registry.h" +#include "table/block_based/block_based_table_factory.h" +#include "table/plain/plain_table_factory.h" +#include "util/cast_util.h" namespace ROCKSDB_NAMESPACE { +// offset_of is used to get the offset of a class data member +// ex: offset_of(&ColumnFamilyOptions::num_levels) +// This call will return the offset of num_levels in ColumnFamilyOptions class +// +// This is the same as offsetof() but allow us to work with non standard-layout +// classes and structures +// refs: +// http://en.cppreference.com/w/cpp/concept/StandardLayoutType +// https://gist.github.com/graphitemaster/494f21190bb2c63c5516 +#ifndef ROCKSDB_LITE +ColumnFamilyOptions OptionsHelper::dummy_cf_options; +template +int offset_of(T1 ColumnFamilyOptions::*member) { + return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - + size_t(&OptionsHelper::dummy_cf_options)); +} +template +int offset_of(T1 AdvancedColumnFamilyOptions::*member) { + return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - + size_t(&OptionsHelper::dummy_cf_options)); +} + +const std::string kNameComparator = "comparator"; +const std::string kNameMergeOperator = "merge_operator"; + +std::unordered_map + OptionsHelper::cf_options_type_info = { + /* not yet supported + CompressionOptions compression_opts; + TablePropertiesCollectorFactories table_properties_collector_factories; + typedef std::vector> + TablePropertiesCollectorFactories; + UpdateStatus (*inplace_callback)(char* existing_value, + uint34_t* existing_value_size, + Slice delta_value, + std::string* merged_value); + std::vector cf_paths; + */ + {"report_bg_io_stats", + {offset_of(&ColumnFamilyOptions::report_bg_io_stats), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, report_bg_io_stats)}}, + {"compaction_measure_io_stats", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"disable_auto_compactions", + {offset_of(&ColumnFamilyOptions::disable_auto_compactions), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, disable_auto_compactions)}}, + {"filter_deletes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"inplace_update_support", + {offset_of(&ColumnFamilyOptions::inplace_update_support), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"level_compaction_dynamic_level_bytes", + {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"optimize_filters_for_hits", + {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"paranoid_file_checks", + {offset_of(&ColumnFamilyOptions::paranoid_file_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, paranoid_file_checks)}}, + {"force_consistency_checks", + {offset_of(&ColumnFamilyOptions::force_consistency_checks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"purge_redundant_kvs_while_flush", + {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush), + OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"verify_checksums_in_compaction", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"soft_pending_compaction_bytes_limit", + {offset_of(&ColumnFamilyOptions::soft_pending_compaction_bytes_limit), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + soft_pending_compaction_bytes_limit)}}, + {"hard_pending_compaction_bytes_limit", + {offset_of(&ColumnFamilyOptions::hard_pending_compaction_bytes_limit), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + hard_pending_compaction_bytes_limit)}}, + {"hard_rate_limit", + {0, OptionType::kDouble, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"soft_rate_limit", + {0, OptionType::kDouble, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"max_compaction_bytes", + {offset_of(&ColumnFamilyOptions::max_compaction_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_compaction_bytes)}}, + {"expanded_compaction_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"level0_file_num_compaction_trigger", + {offset_of(&ColumnFamilyOptions::level0_file_num_compaction_trigger), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + level0_file_num_compaction_trigger)}}, + {"level0_slowdown_writes_trigger", + {offset_of(&ColumnFamilyOptions::level0_slowdown_writes_trigger), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger)}}, + {"level0_stop_writes_trigger", + {offset_of(&ColumnFamilyOptions::level0_stop_writes_trigger), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, level0_stop_writes_trigger)}}, + {"max_grandparent_overlap_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"max_mem_compaction_level", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"max_write_buffer_number", + {offset_of(&ColumnFamilyOptions::max_write_buffer_number), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_write_buffer_number)}}, + {"max_write_buffer_number_to_maintain", + {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_write_buffer_size_to_maintain", + {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain), + OptionType::kInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"min_write_buffer_number_to_merge", + {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"num_levels", + {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"source_compaction_factor", + {0, OptionType::kInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"target_file_size_multiplier", + {offset_of(&ColumnFamilyOptions::target_file_size_multiplier), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, target_file_size_multiplier)}}, + {"arena_block_size", + {offset_of(&ColumnFamilyOptions::arena_block_size), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, arena_block_size)}}, + {"inplace_update_num_locks", + {offset_of(&ColumnFamilyOptions::inplace_update_num_locks), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, inplace_update_num_locks)}}, + {"max_successive_merges", + {offset_of(&ColumnFamilyOptions::max_successive_merges), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_successive_merges)}}, + {"memtable_huge_page_size", + {offset_of(&ColumnFamilyOptions::memtable_huge_page_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, memtable_huge_page_size)}}, + {"memtable_prefix_bloom_huge_page_tlb_size", + {0, OptionType::kSizeT, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"write_buffer_size", + {offset_of(&ColumnFamilyOptions::write_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, write_buffer_size)}}, + {"bloom_locality", + {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"memtable_prefix_bloom_bits", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"memtable_prefix_bloom_size_ratio", + {offset_of(&ColumnFamilyOptions::memtable_prefix_bloom_size_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio)}}, + {"memtable_prefix_bloom_probes", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"memtable_whole_key_filtering", + {offset_of(&ColumnFamilyOptions::memtable_whole_key_filtering), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, memtable_whole_key_filtering)}}, + {"min_partial_merge_operands", + {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"max_bytes_for_level_base", + {offset_of(&ColumnFamilyOptions::max_bytes_for_level_base), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_bytes_for_level_base)}}, + {"snap_refresh_nanos", + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"max_bytes_for_level_multiplier", + {offset_of(&ColumnFamilyOptions::max_bytes_for_level_multiplier), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}}, + {"max_bytes_for_level_multiplier_additional", + {offset_of( + &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional), + OptionType::kVectorInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + max_bytes_for_level_multiplier_additional)}}, + {"max_sequential_skip_in_iterations", + {offset_of(&ColumnFamilyOptions::max_sequential_skip_in_iterations), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, + max_sequential_skip_in_iterations)}}, + {"target_file_size_base", + {offset_of(&ColumnFamilyOptions::target_file_size_base), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, target_file_size_base)}}, + {"rate_limit_delay_max_milliseconds", + {0, OptionType::kUInt, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"compression", + {offset_of(&ColumnFamilyOptions::compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, compression)}}, + {"compression_per_level", + {offset_of(&ColumnFamilyOptions::compression_per_level), + OptionType::kVectorCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"bottommost_compression", + {offset_of(&ColumnFamilyOptions::bottommost_compression), + OptionType::kCompressionType, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, bottommost_compression)}}, + {kNameComparator, + {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator, + OptionVerificationType::kByName, OptionTypeFlags::kNone, 0}}, + {"prefix_extractor", + {offset_of(&ColumnFamilyOptions::prefix_extractor), + OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, prefix_extractor)}}, + {"memtable_insert_with_hint_prefix_extractor", + {offset_of( + &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), + OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, + OptionTypeFlags::kNone, 0}}, + {"memtable_factory", + {offset_of(&ColumnFamilyOptions::memtable_factory), + OptionType::kMemTableRepFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"table_factory", + {offset_of(&ColumnFamilyOptions::table_factory), + OptionType::kTableFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"compaction_filter", + {offset_of(&ColumnFamilyOptions::compaction_filter), + OptionType::kCompactionFilter, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"compaction_filter_factory", + {offset_of(&ColumnFamilyOptions::compaction_filter_factory), + OptionType::kCompactionFilterFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {kNameMergeOperator, + {offset_of(&ColumnFamilyOptions::merge_operator), + OptionType::kMergeOperator, + OptionVerificationType::kByNameAllowFromNull, OptionTypeFlags::kNone, + 0}}, + {"compaction_style", + {offset_of(&ColumnFamilyOptions::compaction_style), + OptionType::kCompactionStyle, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"compaction_pri", + {offset_of(&ColumnFamilyOptions::compaction_pri), + OptionType::kCompactionPri, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"compaction_options_fifo", + {offset_of(&ColumnFamilyOptions::compaction_options_fifo), + OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, compaction_options_fifo)}}, + {"compaction_options_universal", + {offset_of(&ColumnFamilyOptions::compaction_options_universal), + OptionType::kCompactionOptionsUniversal, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, compaction_options_universal)}}, + {"ttl", + {offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, ttl)}}, + {"periodic_compaction_seconds", + {offset_of(&ColumnFamilyOptions::periodic_compaction_seconds), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, periodic_compaction_seconds)}}, + {"sample_for_compression", + {offset_of(&ColumnFamilyOptions::sample_for_compression), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableCFOptions, sample_for_compression)}}}; + +Status ParseColumnFamilyOption(const std::string& name, + const std::string& org_value, + ColumnFamilyOptions* new_options, + bool input_strings_escaped) { + const std::string& value = + input_strings_escaped ? UnescapeOptionString(org_value) : org_value; + try { + if (name == "block_based_table_factory") { + // Nested options + BlockBasedTableOptions table_opt, base_table_options; + BlockBasedTableFactory* block_based_table_factory = + static_cast_with_check( + new_options->table_factory.get()); + if (block_based_table_factory != nullptr) { + base_table_options = block_based_table_factory->table_options(); + } + Status table_opt_s = GetBlockBasedTableOptionsFromString( + base_table_options, value, &table_opt); + if (!table_opt_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt)); + } else if (name == "plain_table_factory") { + // Nested options + PlainTableOptions table_opt, base_table_options; + PlainTableFactory* plain_table_factory = + static_cast_with_check( + new_options->table_factory.get()); + if (plain_table_factory != nullptr) { + base_table_options = plain_table_factory->table_options(); + } + Status table_opt_s = + GetPlainTableOptionsFromString(base_table_options, value, &table_opt); + if (!table_opt_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->table_factory.reset(NewPlainTableFactory(table_opt)); + } else if (name == "memtable") { + std::unique_ptr new_mem_factory; + Status mem_factory_s = + GetMemTableRepFactoryFromString(value, &new_mem_factory); + if (!mem_factory_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->memtable_factory.reset(new_mem_factory.release()); + } else if (name == "bottommost_compression_opts") { + Status s = ParseCompressionOptions( + value, name, new_options->bottommost_compression_opts); + if (!s.ok()) { + return s; + } + } else if (name == "compression_opts") { + Status s = + ParseCompressionOptions(value, name, new_options->compression_opts); + if (!s.ok()) { + return s; + } + } else { + if (name == kNameComparator) { + // Try to get comparator from object registry first. + // Only support static comparator for now. + Status status = ObjectRegistry::NewInstance()->NewStaticObject( + value, &new_options->comparator); + if (status.ok()) { + return status; + } + } else if (name == kNameMergeOperator) { + // Try to get merge operator from object registry first. + std::shared_ptr mo; + Status status = + ObjectRegistry::NewInstance()->NewSharedObject( + value, &new_options->merge_operator); + // Only support static comparator for now. + if (status.ok()) { + return status; + } + } + + auto iter = cf_options_type_info.find(name); + if (iter == cf_options_type_info.end()) { + return Status::InvalidArgument( + "Unable to parse the specified CF option " + name); + } + const auto& opt_info = iter->second; + if (opt_info.IsDeprecated() || + ParseOptionHelper( + reinterpret_cast(new_options) + opt_info.offset, + opt_info.type, value)) { + return Status::OK(); + } else if (opt_info.IsByName()) { + return Status::NotSupported("Deserializing the specified CF option " + + name + " is not supported"); + } else { + return Status::InvalidArgument( + "Unable to parse the specified CF option " + name); + } + } + } catch (const std::exception&) { + return Status::InvalidArgument("unable to parse the specified option " + + name); + } + return Status::OK(); +} +#endif // ROCKSDB_LITE ImmutableCFOptions::ImmutableCFOptions(const Options& options) : ImmutableCFOptions(ImmutableDBOptions(options), options) {} diff --git a/options/db_options.cc b/options/db_options.cc index 8ef147c5e..227525fc7 100644 --- a/options/db_options.cc +++ b/options/db_options.cc @@ -9,6 +9,7 @@ #include "db/version_edit.h" #include "logging/logging.h" +#include "options/options_helper.h" #include "port/port.h" #include "rocksdb/cache.h" #include "rocksdb/env.h" @@ -17,6 +18,321 @@ #include "rocksdb/wal_filter.h" namespace ROCKSDB_NAMESPACE { +#ifndef ROCKSDB_LITE +std::unordered_map + OptionsHelper::db_options_type_info = { + /* + // not yet supported + std::shared_ptr row_cache; + std::shared_ptr delete_scheduler; + std::shared_ptr info_log; + std::shared_ptr rate_limiter; + std::shared_ptr statistics; + std::vector db_paths; + std::vector> listeners; + */ + {"advise_random_on_open", + {offsetof(struct DBOptions, advise_random_on_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"allow_mmap_reads", + {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"allow_fallocate", + {offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"allow_mmap_writes", + {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"use_direct_reads", + {offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"use_direct_writes", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"use_direct_io_for_flush_and_compaction", + {offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"allow_2pc", + {offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"allow_os_buffer", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kMutable, 0}}, + {"create_if_missing", + {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"create_missing_column_families", + {offsetof(struct DBOptions, create_missing_column_families), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"disableDataSync", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"disable_data_sync", // for compatibility + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"enable_thread_tracking", + {offsetof(struct DBOptions, enable_thread_tracking), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"error_if_exists", + {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"is_fd_close_on_exec", + {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"paranoid_checks", + {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"skip_log_error_on_recovery", + {offsetof(struct DBOptions, skip_log_error_on_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"skip_stats_update_on_db_open", + {offsetof(struct DBOptions, skip_stats_update_on_db_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"skip_checking_sst_file_sizes_on_db_open", + {offsetof(struct DBOptions, skip_checking_sst_file_sizes_on_db_open), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"new_table_reader_for_compaction_inputs", + {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"compaction_readahead_size", + {offsetof(struct DBOptions, compaction_readahead_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, compaction_readahead_size)}}, + {"random_access_max_buffer_size", + {offsetof(struct DBOptions, random_access_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"use_adaptive_mutex", + {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"use_fsync", + {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"max_background_jobs", + {offsetof(struct DBOptions, max_background_jobs), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, max_background_jobs)}}, + {"max_background_compactions", + {offsetof(struct DBOptions, max_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, max_background_compactions)}}, + {"base_background_compactions", + {offsetof(struct DBOptions, base_background_compactions), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, base_background_compactions)}}, + {"max_background_flushes", + {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"max_file_opening_threads", + {offsetof(struct DBOptions, max_file_opening_threads), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_open_files", + {offsetof(struct DBOptions, max_open_files), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, max_open_files)}}, + {"table_cache_numshardbits", + {offsetof(struct DBOptions, table_cache_numshardbits), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"db_write_buffer_size", + {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"keep_log_file_num", + {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"recycle_log_file_num", + {offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"log_file_time_to_roll", + {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"manifest_preallocation_size", + {offsetof(struct DBOptions, manifest_preallocation_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_log_file_size", + {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"db_log_dir", + {offsetof(struct DBOptions, db_log_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"wal_dir", + {offsetof(struct DBOptions, wal_dir), OptionType::kString, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"max_subcompactions", + {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"WAL_size_limit_MB", + {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"WAL_ttl_seconds", + {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"bytes_per_sync", + {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, bytes_per_sync)}}, + {"delayed_write_rate", + {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, delayed_write_rate)}}, + {"delete_obsolete_files_period_micros", + {offsetof(struct DBOptions, delete_obsolete_files_period_micros), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, + delete_obsolete_files_period_micros)}}, + {"max_manifest_file_size", + {offsetof(struct DBOptions, max_manifest_file_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_total_wal_size", + {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, max_total_wal_size)}}, + {"wal_bytes_per_sync", + {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, wal_bytes_per_sync)}}, + {"strict_bytes_per_sync", + {offsetof(struct DBOptions, strict_bytes_per_sync), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, strict_bytes_per_sync)}}, + {"stats_dump_period_sec", + {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, stats_dump_period_sec)}}, + {"stats_persist_period_sec", + {offsetof(struct DBOptions, stats_persist_period_sec), + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, stats_persist_period_sec)}}, + {"persist_stats_to_disk", + {offsetof(struct DBOptions, persist_stats_to_disk), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}}, + {"stats_history_buffer_size", + {offsetof(struct DBOptions, stats_history_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, stats_history_buffer_size)}}, + {"fail_if_options_file_error", + {offsetof(struct DBOptions, fail_if_options_file_error), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"enable_pipelined_write", + {offsetof(struct DBOptions, enable_pipelined_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"unordered_write", + {offsetof(struct DBOptions, unordered_write), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"allow_concurrent_memtable_write", + {offsetof(struct DBOptions, allow_concurrent_memtable_write), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"wal_recovery_mode", + {offsetof(struct DBOptions, wal_recovery_mode), + OptionType::kWALRecoveryMode, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"enable_write_thread_adaptive_yield", + {offsetof(struct DBOptions, enable_write_thread_adaptive_yield), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"write_thread_slow_yield_usec", + {offsetof(struct DBOptions, write_thread_slow_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"max_write_batch_group_size_bytes", + {offsetof(struct DBOptions, max_write_batch_group_size_bytes), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"write_thread_max_yield_usec", + {offsetof(struct DBOptions, write_thread_max_yield_usec), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"access_hint_on_compaction_start", + {offsetof(struct DBOptions, access_hint_on_compaction_start), + OptionType::kAccessHint, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"info_log_level", + {offsetof(struct DBOptions, info_log_level), OptionType::kInfoLogLevel, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"dump_malloc_stats", + {offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"avoid_flush_during_recovery", + {offsetof(struct DBOptions, avoid_flush_during_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"avoid_flush_during_shutdown", + {offsetof(struct DBOptions, avoid_flush_during_shutdown), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}}, + {"writable_file_max_buffer_size", + {offsetof(struct DBOptions, writable_file_max_buffer_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, + offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}}, + {"allow_ingest_behind", + {offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, allow_ingest_behind)}}, + {"preserve_deletes", + {offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, preserve_deletes)}}, + {"concurrent_prepare", // Deprecated by two_write_queues + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"two_write_queues", + {offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, two_write_queues)}}, + {"manual_wal_flush", + {offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, manual_wal_flush)}}, + {"seq_per_batch", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"atomic_flush", + {offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, atomic_flush)}}, + {"avoid_unnecessary_blocking_io", + {offsetof(struct DBOptions, avoid_unnecessary_blocking_io), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, + offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}}, + {"write_dbid_to_manifest", + {offsetof(struct DBOptions, write_dbid_to_manifest), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"log_readahead_size", + {offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"best_efforts_recovery", + {offsetof(struct DBOptions, best_efforts_recovery), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, +}; +#endif // ROCKSDB_LITE ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {} diff --git a/options/options_helper.cc b/options/options_helper.cc index 4a9c0fe96..f3ee13ad8 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -23,7 +23,6 @@ #include "rocksdb/utilities/object_registry.h" #include "table/block_based/block_based_table_factory.h" #include "table/plain/plain_table_factory.h" -#include "util/cast_util.h" #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { @@ -256,18 +255,10 @@ std::unordered_map {"kDisableCompressionOption", kDisableCompressionOption}}; #ifndef ROCKSDB_LITE -const std::string kNameComparator = "comparator"; const std::string kNameEnv = "env"; -const std::string kNameMergeOperator = "merge_operator"; const std::string kOptNameBMCompOpts = "bottommost_compression_opts"; const std::string kOptNameCompOpts = "compression_opts"; -template -Status GetStringFromStruct( - std::string* opt_string, const T& options, - const std::unordered_map& type_info, - const std::string& delimiter); - namespace { template bool ParseEnum(const std::unordered_map& type_map, @@ -357,9 +348,8 @@ bool FIFOCompactionOptionsSpecialCase(const std::string& opt_str, return true; } -template -bool SerializeStruct( - const T& options, std::string* value, +static bool SerializeStruct( + const void* const options, std::string* value, const std::unordered_map& type_info_map) { std::string opt_str; Status s = GetStringFromStruct(&opt_str, options, type_info_map, ";"); @@ -370,9 +360,8 @@ bool SerializeStruct( return true; } -template -bool ParseSingleStructOption( - const std::string& opt_val_str, T* options, +static bool ParseSingleStructOption( + const std::string& opt_val_str, void* options, const std::unordered_map& type_info_map) { size_t end = opt_val_str.find('='); std::string key = opt_val_str.substr(0, end); @@ -382,7 +371,7 @@ bool ParseSingleStructOption( return false; } const auto& opt_info = iter->second; - if (opt_info.verification == OptionVerificationType::kDeprecated) { + if (opt_info.IsDeprecated()) { // Should also skip deprecated sub-options such as // fifo_compaction_options_type_info.ttl return true; @@ -392,9 +381,8 @@ bool ParseSingleStructOption( value); } -template -bool ParseStructOptions( - const std::string& opt_str, T* options, +static bool ParseStructOptions( + const std::string& opt_str, void* options, const std::unordered_map& type_info_map) { assert(!opt_str.empty()); @@ -565,21 +553,18 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, case OptionType::kCompactionOptionsFIFO: { if (!FIFOCompactionOptionsSpecialCase( value, reinterpret_cast(opt_address))) { - return ParseStructOptions( - value, reinterpret_cast(opt_address), - fifo_compaction_options_type_info); + return ParseStructOptions(value, opt_address, + fifo_compaction_options_type_info); } return true; } case OptionType::kLRUCacheOptions: { - return ParseStructOptions(value, - reinterpret_cast(opt_address), - lru_cache_options_type_info); + return ParseStructOptions(value, opt_address, + lru_cache_options_type_info); } case OptionType::kCompactionOptionsUniversal: - return ParseStructOptions( - value, reinterpret_cast(opt_address), - universal_compaction_options_type_info); + return ParseStructOptions(value, opt_address, + universal_compaction_options_type_info); case OptionType::kCompactionStopStyle: return ParseEnum( compaction_stop_style_string_map, value, @@ -771,13 +756,11 @@ bool SerializeSingleOptionHelper(const char* opt_address, info_log_level_string_map, *reinterpret_cast(opt_address), value); case OptionType::kCompactionOptionsFIFO: - return SerializeStruct( - *reinterpret_cast(opt_address), value, - fifo_compaction_options_type_info); + return SerializeStruct(opt_address, value, + fifo_compaction_options_type_info); case OptionType::kCompactionOptionsUniversal: - return SerializeStruct( - *reinterpret_cast(opt_address), - value, universal_compaction_options_type_info); + return SerializeStruct(opt_address, value, + universal_compaction_options_type_info); case OptionType::kCompactionStopStyle: return SerializeEnum( compaction_stop_style_string_map, @@ -889,11 +872,11 @@ Status GetMutableOptionsFromStrings( return Status::InvalidArgument("Unrecognized option: " + option_name); } const auto& opt_info = iter->second; - if (!opt_info.is_mutable) { + if (!opt_info.IsMutable()) { return Status::InvalidArgument("Option not changeable: " + option_name); } - if (opt_info.verification == OptionVerificationType::kDeprecated) { + if (opt_info.IsDeprecated()) { // log warning when user tries to set a deprecated option but don't // fail the call for compatibility. ROCKS_LOG_WARN(info_log, @@ -929,7 +912,7 @@ Status GetMutableDBOptionsFromStrings( return Status::InvalidArgument("Unrecognized option: " + o.first); } const auto& opt_info = iter->second; - if (!opt_info.is_mutable) { + if (!opt_info.IsMutable()) { return Status::InvalidArgument("Option not changeable: " + o.first); } bool is_ok = ParseOptionHelper( @@ -1022,163 +1005,30 @@ Status StringToMap(const std::string& opts_str, return Status::OK(); } -Status ParseColumnFamilyOption(const std::string& name, - const std::string& org_value, - ColumnFamilyOptions* new_options, - bool input_strings_escaped = false) { - const std::string& value = - input_strings_escaped ? UnescapeOptionString(org_value) : org_value; - try { - if (name == "block_based_table_factory") { - // Nested options - BlockBasedTableOptions table_opt, base_table_options; - BlockBasedTableFactory* block_based_table_factory = - static_cast_with_check( - new_options->table_factory.get()); - if (block_based_table_factory != nullptr) { - base_table_options = block_based_table_factory->table_options(); - } - Status table_opt_s = GetBlockBasedTableOptionsFromString( - base_table_options, value, &table_opt); - if (!table_opt_s.ok()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt)); - } else if (name == "plain_table_factory") { - // Nested options - PlainTableOptions table_opt, base_table_options; - PlainTableFactory* plain_table_factory = - static_cast_with_check( - new_options->table_factory.get()); - if (plain_table_factory != nullptr) { - base_table_options = plain_table_factory->table_options(); - } - Status table_opt_s = GetPlainTableOptionsFromString( - base_table_options, value, &table_opt); - if (!table_opt_s.ok()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - new_options->table_factory.reset(NewPlainTableFactory(table_opt)); - } else if (name == "memtable") { - std::unique_ptr new_mem_factory; - Status mem_factory_s = - GetMemTableRepFactoryFromString(value, &new_mem_factory); - if (!mem_factory_s.ok()) { - return Status::InvalidArgument( - "unable to parse the specified CF option " + name); - } - new_options->memtable_factory.reset(new_mem_factory.release()); - } else if (name == "bottommost_compression_opts") { - Status s = ParseCompressionOptions( - value, name, new_options->bottommost_compression_opts); - if (!s.ok()) { - return s; - } - } else if (name == "compression_opts") { - Status s = - ParseCompressionOptions(value, name, new_options->compression_opts); - if (!s.ok()) { - return s; - } - } else { - if (name == kNameComparator) { - // Try to get comparator from object registry first. - // Only support static comparator for now. - Status status = ObjectRegistry::NewInstance()->NewStaticObject( - value, &new_options->comparator); - if (status.ok()) { - return status; - } - } else if (name == kNameMergeOperator) { - // Try to get merge operator from object registry first. - std::shared_ptr mo; - Status status = - ObjectRegistry::NewInstance()->NewSharedObject( - value, &new_options->merge_operator); - // Only support static comparator for now. - if (status.ok()) { - return status; - } - } - - auto iter = cf_options_type_info.find(name); - if (iter == cf_options_type_info.end()) { - return Status::InvalidArgument( - "Unable to parse the specified CF option " + name); - } - const auto& opt_info = iter->second; - if (opt_info.verification != OptionVerificationType::kDeprecated && - ParseOptionHelper( - reinterpret_cast(new_options) + opt_info.offset, - opt_info.type, value)) { - return Status::OK(); - } - switch (opt_info.verification) { - case OptionVerificationType::kByName: - case OptionVerificationType::kByNameAllowNull: - case OptionVerificationType::kByNameAllowFromNull: - return Status::NotSupported( - "Deserializing the specified CF option " + name + - " is not supported"); - case OptionVerificationType::kDeprecated: - return Status::OK(); - default: - return Status::InvalidArgument( - "Unable to parse the specified CF option " + name); - } - } - } catch (const std::exception&) { - return Status::InvalidArgument( - "unable to parse the specified option " + name); - } - return Status::OK(); -} - -template -bool SerializeSingleStructOption( - std::string* opt_string, const T& options, - const std::unordered_map& type_info, - const std::string& name, const std::string& delimiter) { - auto iter = type_info.find(name); - if (iter == type_info.end()) { - return false; - } - auto& opt_info = iter->second; - const char* opt_address = - reinterpret_cast(&options) + opt_info.offset; - std::string value; - bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value); - if (result) { - *opt_string = name + "=" + value + delimiter; - } - return result; -} - -template Status GetStringFromStruct( - std::string* opt_string, const T& options, + std::string* opt_string, const void* const options, const std::unordered_map& type_info, const std::string& delimiter) { assert(opt_string); opt_string->clear(); - for (auto iter = type_info.begin(); iter != type_info.end(); ++iter) { - if (iter->second.verification == OptionVerificationType::kDeprecated) { + for (const auto iter : type_info) { + const auto& opt_info = iter.second; + if (opt_info.IsDeprecated()) { // If the option is no longer used in rocksdb and marked as deprecated, // we skip it in the serialization. continue; } - std::string single_output; - bool result = SerializeSingleStructOption( - &single_output, options, type_info, iter->first, delimiter); + const char* opt_address = + reinterpret_cast(options) + opt_info.offset; + std::string value; + bool result = + SerializeSingleOptionHelper(opt_address, opt_info.type, &value); if (result) { - opt_string->append(single_output); + opt_string->append(iter.first + "=" + value + delimiter); } else { return Status::InvalidArgument("failed to serialize %s\n", - iter->first.c_str()); + iter.first.c_str()); } - assert(result); } return Status::OK(); } @@ -1186,15 +1036,15 @@ Status GetStringFromStruct( Status GetStringFromDBOptions(std::string* opt_string, const DBOptions& db_options, const std::string& delimiter) { - return GetStringFromStruct(opt_string, db_options, - db_options_type_info, delimiter); + return GetStringFromStruct(opt_string, &db_options, db_options_type_info, + delimiter); } Status GetStringFromColumnFamilyOptions(std::string* opt_string, const ColumnFamilyOptions& cf_options, const std::string& delimiter) { - return GetStringFromStruct( - opt_string, cf_options, cf_options_type_info, delimiter); + return GetStringFromStruct(opt_string, &cf_options, cf_options_type_info, + delimiter); } Status GetStringFromCompressionType(std::string* compression_str, @@ -1243,23 +1093,17 @@ Status ParseDBOption(const std::string& name, return Status::InvalidArgument("Unrecognized option DBOptions:", name); } const auto& opt_info = iter->second; - if (opt_info.verification != OptionVerificationType::kDeprecated && + if (opt_info.IsDeprecated() || ParseOptionHelper( reinterpret_cast(new_options) + opt_info.offset, opt_info.type, value)) { return Status::OK(); - } - switch (opt_info.verification) { - case OptionVerificationType::kByName: - case OptionVerificationType::kByNameAllowNull: - return Status::NotSupported( - "Deserializing the specified DB option " + name + - " is not supported"); - case OptionVerificationType::kDeprecated: - return Status::OK(); - default: - return Status::InvalidArgument( - "Unable to parse the specified DB option " + name); + } else if (opt_info.IsByName()) { + return Status::NotSupported("Deserializing the specified DB option " + + name + " is not supported"); + } else { + return Status::InvalidArgument( + "Unable to parse the specified DB option " + name); } } } catch (const std::exception&) { @@ -1442,285 +1286,6 @@ Status GetTableFactoryFromMap( return Status::OK(); } -std::unordered_map - OptionsHelper::db_options_type_info = { - /* - // not yet supported - std::shared_ptr row_cache; - std::shared_ptr delete_scheduler; - std::shared_ptr info_log; - std::shared_ptr rate_limiter; - std::shared_ptr statistics; - std::vector db_paths; - std::vector> listeners; - */ - {"advise_random_on_open", - {offsetof(struct DBOptions, advise_random_on_open), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"allow_mmap_reads", - {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"allow_fallocate", - {offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"allow_mmap_writes", - {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"use_direct_reads", - {offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"use_direct_writes", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"use_direct_io_for_flush_and_compaction", - {offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"allow_2pc", - {offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"allow_os_buffer", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, - 0}}, - {"create_if_missing", - {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"create_missing_column_families", - {offsetof(struct DBOptions, create_missing_column_families), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"disableDataSync", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"disable_data_sync", // for compatibility - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"enable_thread_tracking", - {offsetof(struct DBOptions, enable_thread_tracking), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"error_if_exists", - {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"is_fd_close_on_exec", - {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"paranoid_checks", - {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"skip_log_error_on_recovery", - {offsetof(struct DBOptions, skip_log_error_on_recovery), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"skip_stats_update_on_db_open", - {offsetof(struct DBOptions, skip_stats_update_on_db_open), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"skip_checking_sst_file_sizes_on_db_open", - {offsetof(struct DBOptions, skip_checking_sst_file_sizes_on_db_open), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"new_table_reader_for_compaction_inputs", - {offsetof(struct DBOptions, new_table_reader_for_compaction_inputs), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"compaction_readahead_size", - {offsetof(struct DBOptions, compaction_readahead_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, compaction_readahead_size)}}, - {"random_access_max_buffer_size", - {offsetof(struct DBOptions, random_access_max_buffer_size), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"use_adaptive_mutex", - {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"use_fsync", - {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"max_background_jobs", - {offsetof(struct DBOptions, max_background_jobs), OptionType::kInt, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, max_background_jobs)}}, - {"max_background_compactions", - {offsetof(struct DBOptions, max_background_compactions), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, max_background_compactions)}}, - {"base_background_compactions", - {offsetof(struct DBOptions, base_background_compactions), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, base_background_compactions)}}, - {"max_background_flushes", - {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt, - OptionVerificationType::kNormal, false, 0}}, - {"max_file_opening_threads", - {offsetof(struct DBOptions, max_file_opening_threads), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"max_open_files", - {offsetof(struct DBOptions, max_open_files), OptionType::kInt, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, max_open_files)}}, - {"table_cache_numshardbits", - {offsetof(struct DBOptions, table_cache_numshardbits), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"db_write_buffer_size", - {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"keep_log_file_num", - {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"recycle_log_file_num", - {offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"log_file_time_to_roll", - {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"manifest_preallocation_size", - {offsetof(struct DBOptions, manifest_preallocation_size), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"max_log_file_size", - {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"db_log_dir", - {offsetof(struct DBOptions, db_log_dir), OptionType::kString, - OptionVerificationType::kNormal, false, 0}}, - {"wal_dir", - {offsetof(struct DBOptions, wal_dir), OptionType::kString, - OptionVerificationType::kNormal, false, 0}}, - {"max_subcompactions", - {offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T, - OptionVerificationType::kNormal, false, 0}}, - {"WAL_size_limit_MB", - {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T, - OptionVerificationType::kNormal, false, 0}}, - {"WAL_ttl_seconds", - {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T, - OptionVerificationType::kNormal, false, 0}}, - {"bytes_per_sync", - {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, bytes_per_sync)}}, - {"delayed_write_rate", - {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, delayed_write_rate)}}, - {"delete_obsolete_files_period_micros", - {offsetof(struct DBOptions, delete_obsolete_files_period_micros), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, - delete_obsolete_files_period_micros)}}, - {"max_manifest_file_size", - {offsetof(struct DBOptions, max_manifest_file_size), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"max_total_wal_size", - {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, max_total_wal_size)}}, - {"wal_bytes_per_sync", - {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, wal_bytes_per_sync)}}, - {"strict_bytes_per_sync", - {offsetof(struct DBOptions, strict_bytes_per_sync), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, strict_bytes_per_sync)}}, - {"stats_dump_period_sec", - {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt, - OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, stats_dump_period_sec)}}, - {"stats_persist_period_sec", - {offsetof(struct DBOptions, stats_persist_period_sec), - OptionType::kUInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, stats_persist_period_sec)}}, - {"persist_stats_to_disk", - {offsetof(struct DBOptions, persist_stats_to_disk), - OptionType::kBoolean, OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}}, - {"stats_history_buffer_size", - {offsetof(struct DBOptions, stats_history_buffer_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, stats_history_buffer_size)}}, - {"fail_if_options_file_error", - {offsetof(struct DBOptions, fail_if_options_file_error), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"enable_pipelined_write", - {offsetof(struct DBOptions, enable_pipelined_write), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"unordered_write", - {offsetof(struct DBOptions, unordered_write), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"allow_concurrent_memtable_write", - {offsetof(struct DBOptions, allow_concurrent_memtable_write), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"wal_recovery_mode", - {offsetof(struct DBOptions, wal_recovery_mode), - OptionType::kWALRecoveryMode, OptionVerificationType::kNormal, false, - 0}}, - {"enable_write_thread_adaptive_yield", - {offsetof(struct DBOptions, enable_write_thread_adaptive_yield), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"write_thread_slow_yield_usec", - {offsetof(struct DBOptions, write_thread_slow_yield_usec), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"max_write_batch_group_size_bytes", - {offsetof(struct DBOptions, max_write_batch_group_size_bytes), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"write_thread_max_yield_usec", - {offsetof(struct DBOptions, write_thread_max_yield_usec), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"access_hint_on_compaction_start", - {offsetof(struct DBOptions, access_hint_on_compaction_start), - OptionType::kAccessHint, OptionVerificationType::kNormal, false, 0}}, - {"info_log_level", - {offsetof(struct DBOptions, info_log_level), OptionType::kInfoLogLevel, - OptionVerificationType::kNormal, false, 0}}, - {"dump_malloc_stats", - {offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"avoid_flush_during_recovery", - {offsetof(struct DBOptions, avoid_flush_during_recovery), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"avoid_flush_during_shutdown", - {offsetof(struct DBOptions, avoid_flush_during_shutdown), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}}, - {"writable_file_max_buffer_size", - {offsetof(struct DBOptions, writable_file_max_buffer_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}}, - {"allow_ingest_behind", - {offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, allow_ingest_behind)}}, - {"preserve_deletes", - {offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, preserve_deletes)}}, - {"concurrent_prepare", // Deprecated by two_write_queues - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"two_write_queues", - {offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, two_write_queues)}}, - {"manual_wal_flush", - {offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, manual_wal_flush)}}, - {"seq_per_batch", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"atomic_flush", - {offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean, - OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, atomic_flush)}}, - {"avoid_unnecessary_blocking_io", - {offsetof(struct DBOptions, avoid_unnecessary_blocking_io), - OptionType::kBoolean, OptionVerificationType::kNormal, false, - offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}}, - {"write_dbid_to_manifest", - {offsetof(struct DBOptions, write_dbid_to_manifest), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"log_readahead_size", - {offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"best_efforts_recovery", - {offsetof(struct DBOptions, best_efforts_recovery), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, -}; - std::unordered_map OptionsHelper::block_base_table_index_type_string_map = { {"kBinarySearch", BlockBasedTableOptions::IndexType::kBinarySearch}, @@ -1790,347 +1355,81 @@ std::unordered_map {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL}, {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}}; -ColumnFamilyOptions OptionsHelper::dummy_cf_options; -CompactionOptionsFIFO OptionsHelper::dummy_comp_options; LRUCacheOptions OptionsHelper::dummy_lru_cache_options; CompactionOptionsUniversal OptionsHelper::dummy_comp_options_universal; +CompactionOptionsFIFO OptionsHelper::dummy_comp_options; -// offset_of is used to get the offset of a class data member -// ex: offset_of(&ColumnFamilyOptions::num_levels) -// This call will return the offset of num_levels in ColumnFamilyOptions class -// -// This is the same as offsetof() but allow us to work with non standard-layout -// classes and structures -// refs: -// http://en.cppreference.com/w/cpp/concept/StandardLayoutType -// https://gist.github.com/graphitemaster/494f21190bb2c63c5516 -template -int offset_of(T1 ColumnFamilyOptions::*member) { - return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - - size_t(&OptionsHelper::dummy_cf_options)); -} template -int offset_of(T1 AdvancedColumnFamilyOptions::*member) { - return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) - - size_t(&OptionsHelper::dummy_cf_options)); +int offset_of(T1 LRUCacheOptions::*member) { + return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) - + size_t(&OptionsHelper::dummy_lru_cache_options)); } + template int offset_of(T1 CompactionOptionsFIFO::*member) { return int(size_t(&(OptionsHelper::dummy_comp_options.*member)) - size_t(&OptionsHelper::dummy_comp_options)); } template -int offset_of(T1 LRUCacheOptions::*member) { - return int(size_t(&(OptionsHelper::dummy_lru_cache_options.*member)) - - size_t(&OptionsHelper::dummy_lru_cache_options)); -} -template int offset_of(T1 CompactionOptionsUniversal::*member) { return int(size_t(&(OptionsHelper::dummy_comp_options_universal.*member)) - size_t(&OptionsHelper::dummy_comp_options_universal)); } -std::unordered_map - OptionsHelper::cf_options_type_info = { - /* not yet supported - CompressionOptions compression_opts; - TablePropertiesCollectorFactories table_properties_collector_factories; - typedef std::vector> - TablePropertiesCollectorFactories; - UpdateStatus (*inplace_callback)(char* existing_value, - uint34_t* existing_value_size, - Slice delta_value, - std::string* merged_value); - std::vector cf_paths; - */ - {"report_bg_io_stats", - {offset_of(&ColumnFamilyOptions::report_bg_io_stats), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, report_bg_io_stats)}}, - {"compaction_measure_io_stats", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"disable_auto_compactions", - {offset_of(&ColumnFamilyOptions::disable_auto_compactions), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, disable_auto_compactions)}}, - {"filter_deletes", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, - 0}}, - {"inplace_update_support", - {offset_of(&ColumnFamilyOptions::inplace_update_support), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"level_compaction_dynamic_level_bytes", - {offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"optimize_filters_for_hits", - {offset_of(&ColumnFamilyOptions::optimize_filters_for_hits), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"paranoid_file_checks", - {offset_of(&ColumnFamilyOptions::paranoid_file_checks), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, paranoid_file_checks)}}, - {"force_consistency_checks", - {offset_of(&ColumnFamilyOptions::force_consistency_checks), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"purge_redundant_kvs_while_flush", - {offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush), - OptionType::kBoolean, OptionVerificationType::kDeprecated, false, 0}}, - {"verify_checksums_in_compaction", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, true, - 0}}, - {"soft_pending_compaction_bytes_limit", - {offset_of(&ColumnFamilyOptions::soft_pending_compaction_bytes_limit), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - soft_pending_compaction_bytes_limit)}}, - {"hard_pending_compaction_bytes_limit", - {offset_of(&ColumnFamilyOptions::hard_pending_compaction_bytes_limit), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - hard_pending_compaction_bytes_limit)}}, - {"hard_rate_limit", - {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true, - 0}}, - {"soft_rate_limit", - {0, OptionType::kDouble, OptionVerificationType::kDeprecated, true, - 0}}, - {"max_compaction_bytes", - {offset_of(&ColumnFamilyOptions::max_compaction_bytes), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_compaction_bytes)}}, - {"expanded_compaction_factor", - {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}}, - {"level0_file_num_compaction_trigger", - {offset_of(&ColumnFamilyOptions::level0_file_num_compaction_trigger), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - level0_file_num_compaction_trigger)}}, - {"level0_slowdown_writes_trigger", - {offset_of(&ColumnFamilyOptions::level0_slowdown_writes_trigger), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger)}}, - {"level0_stop_writes_trigger", - {offset_of(&ColumnFamilyOptions::level0_stop_writes_trigger), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, level0_stop_writes_trigger)}}, - {"max_grandparent_overlap_factor", - {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}}, - {"max_mem_compaction_level", - {0, OptionType::kInt, OptionVerificationType::kDeprecated, false, 0}}, - {"max_write_buffer_number", - {offset_of(&ColumnFamilyOptions::max_write_buffer_number), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_write_buffer_number)}}, - {"max_write_buffer_number_to_maintain", - {offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"max_write_buffer_size_to_maintain", - {offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain), - OptionType::kInt64T, OptionVerificationType::kNormal, false, 0}}, - {"min_write_buffer_number_to_merge", - {offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"num_levels", - {offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt, - OptionVerificationType::kNormal, false, 0}}, - {"source_compaction_factor", - {0, OptionType::kInt, OptionVerificationType::kDeprecated, true, 0}}, - {"target_file_size_multiplier", - {offset_of(&ColumnFamilyOptions::target_file_size_multiplier), - OptionType::kInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, target_file_size_multiplier)}}, - {"arena_block_size", - {offset_of(&ColumnFamilyOptions::arena_block_size), OptionType::kSizeT, - OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, arena_block_size)}}, - {"inplace_update_num_locks", - {offset_of(&ColumnFamilyOptions::inplace_update_num_locks), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, inplace_update_num_locks)}}, - {"max_successive_merges", - {offset_of(&ColumnFamilyOptions::max_successive_merges), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_successive_merges)}}, - {"memtable_huge_page_size", - {offset_of(&ColumnFamilyOptions::memtable_huge_page_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, memtable_huge_page_size)}}, - {"memtable_prefix_bloom_huge_page_tlb_size", - {0, OptionType::kSizeT, OptionVerificationType::kDeprecated, true, 0}}, - {"write_buffer_size", - {offset_of(&ColumnFamilyOptions::write_buffer_size), - OptionType::kSizeT, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, write_buffer_size)}}, - {"bloom_locality", - {offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T, - OptionVerificationType::kNormal, false, 0}}, - {"memtable_prefix_bloom_bits", - {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, - 0}}, - {"memtable_prefix_bloom_size_ratio", - {offset_of(&ColumnFamilyOptions::memtable_prefix_bloom_size_ratio), - OptionType::kDouble, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio)}}, - {"memtable_prefix_bloom_probes", - {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, - 0}}, - {"memtable_whole_key_filtering", - {offset_of(&ColumnFamilyOptions::memtable_whole_key_filtering), - OptionType::kBoolean, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, memtable_whole_key_filtering)}}, - {"min_partial_merge_operands", - {0, OptionType::kUInt32T, OptionVerificationType::kDeprecated, true, - 0}}, - {"max_bytes_for_level_base", - {offset_of(&ColumnFamilyOptions::max_bytes_for_level_base), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_bytes_for_level_base)}}, - {"snap_refresh_nanos", - {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, true, - 0}}, - {"max_bytes_for_level_multiplier", - {offset_of(&ColumnFamilyOptions::max_bytes_for_level_multiplier), - OptionType::kDouble, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}}, - {"max_bytes_for_level_multiplier_additional", - {offset_of( - &ColumnFamilyOptions::max_bytes_for_level_multiplier_additional), - OptionType::kVectorInt, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - max_bytes_for_level_multiplier_additional)}}, - {"max_sequential_skip_in_iterations", - {offset_of(&ColumnFamilyOptions::max_sequential_skip_in_iterations), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, - max_sequential_skip_in_iterations)}}, - {"target_file_size_base", - {offset_of(&ColumnFamilyOptions::target_file_size_base), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, target_file_size_base)}}, - {"rate_limit_delay_max_milliseconds", - {0, OptionType::kUInt, OptionVerificationType::kDeprecated, false, 0}}, - {"compression", - {offset_of(&ColumnFamilyOptions::compression), - OptionType::kCompressionType, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, compression)}}, - {"compression_per_level", - {offset_of(&ColumnFamilyOptions::compression_per_level), - OptionType::kVectorCompressionType, OptionVerificationType::kNormal, - false, 0}}, - {"bottommost_compression", - {offset_of(&ColumnFamilyOptions::bottommost_compression), - OptionType::kCompressionType, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, bottommost_compression)}}, - {kNameComparator, - {offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator, - OptionVerificationType::kByName, false, 0}}, - {"prefix_extractor", - {offset_of(&ColumnFamilyOptions::prefix_extractor), - OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, - true, offsetof(struct MutableCFOptions, prefix_extractor)}}, - {"memtable_insert_with_hint_prefix_extractor", - {offset_of( - &ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor), - OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull, - false, 0}}, - {"memtable_factory", - {offset_of(&ColumnFamilyOptions::memtable_factory), - OptionType::kMemTableRepFactory, OptionVerificationType::kByName, - false, 0}}, - {"table_factory", - {offset_of(&ColumnFamilyOptions::table_factory), - OptionType::kTableFactory, OptionVerificationType::kByName, false, - 0}}, - {"compaction_filter", - {offset_of(&ColumnFamilyOptions::compaction_filter), - OptionType::kCompactionFilter, OptionVerificationType::kByName, false, - 0}}, - {"compaction_filter_factory", - {offset_of(&ColumnFamilyOptions::compaction_filter_factory), - OptionType::kCompactionFilterFactory, OptionVerificationType::kByName, - false, 0}}, - {kNameMergeOperator, - {offset_of(&ColumnFamilyOptions::merge_operator), - OptionType::kMergeOperator, - OptionVerificationType::kByNameAllowFromNull, false, 0}}, - {"compaction_style", - {offset_of(&ColumnFamilyOptions::compaction_style), - OptionType::kCompactionStyle, OptionVerificationType::kNormal, false, - 0}}, - {"compaction_pri", - {offset_of(&ColumnFamilyOptions::compaction_pri), - OptionType::kCompactionPri, OptionVerificationType::kNormal, false, - 0}}, - {"compaction_options_fifo", - {offset_of(&ColumnFamilyOptions::compaction_options_fifo), - OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal, - true, offsetof(struct MutableCFOptions, compaction_options_fifo)}}, - {"compaction_options_universal", - {offset_of(&ColumnFamilyOptions::compaction_options_universal), - OptionType::kCompactionOptionsUniversal, - OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, compaction_options_universal)}}, - {"ttl", - {offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T, - OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, ttl)}}, - {"periodic_compaction_seconds", - {offset_of(&ColumnFamilyOptions::periodic_compaction_seconds), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, periodic_compaction_seconds)}}, - {"sample_for_compression", - {offset_of(&ColumnFamilyOptions::sample_for_compression), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, - offsetof(struct MutableCFOptions, sample_for_compression)}}}; - std::unordered_map OptionsHelper::fifo_compaction_options_type_info = { {"max_table_files_size", {offset_of(&CompactionOptionsFIFO::max_table_files_size), - OptionType::kUInt64T, OptionVerificationType::kNormal, true, + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(struct CompactionOptionsFIFO, max_table_files_size)}}, {"ttl", - {0, OptionType::kUInt64T, - OptionVerificationType::kDeprecated, false, - 0}}, + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, {"allow_compaction", {offset_of(&CompactionOptionsFIFO::allow_compaction), - OptionType::kBoolean, OptionVerificationType::kNormal, true, + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(struct CompactionOptionsFIFO, allow_compaction)}}}; std::unordered_map OptionsHelper::universal_compaction_options_type_info = { {"size_ratio", {offset_of(&CompactionOptionsUniversal::size_ratio), OptionType::kUInt, - OptionVerificationType::kNormal, true, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, size_ratio)}}, {"min_merge_width", {offset_of(&CompactionOptionsUniversal::min_merge_width), - OptionType::kUInt, OptionVerificationType::kNormal, true, + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, min_merge_width)}}, {"max_merge_width", {offset_of(&CompactionOptionsUniversal::max_merge_width), - OptionType::kUInt, OptionVerificationType::kNormal, true, + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, max_merge_width)}}, {"max_size_amplification_percent", {offset_of( &CompactionOptionsUniversal::max_size_amplification_percent), - OptionType::kUInt, OptionVerificationType::kNormal, true, + OptionType::kUInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, max_size_amplification_percent)}}, {"compression_size_percent", {offset_of(&CompactionOptionsUniversal::compression_size_percent), - OptionType::kInt, OptionVerificationType::kNormal, true, + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, compression_size_percent)}}, {"stop_style", {offset_of(&CompactionOptionsUniversal::stop_style), OptionType::kCompactionStopStyle, OptionVerificationType::kNormal, - true, offsetof(class CompactionOptionsUniversal, stop_style)}}, + OptionTypeFlags::kMutable, + offsetof(class CompactionOptionsUniversal, stop_style)}}, {"allow_trivial_move", {offset_of(&CompactionOptionsUniversal::allow_trivial_move), - OptionType::kBoolean, OptionVerificationType::kNormal, true, + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(class CompactionOptionsUniversal, allow_trivial_move)}}}; std::unordered_map @@ -2142,19 +1441,20 @@ std::unordered_map OptionsHelper::lru_cache_options_type_info = { {"capacity", {offset_of(&LRUCacheOptions::capacity), OptionType::kSizeT, - OptionVerificationType::kNormal, true, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct LRUCacheOptions, capacity)}}, {"num_shard_bits", {offset_of(&LRUCacheOptions::num_shard_bits), OptionType::kInt, - OptionVerificationType::kNormal, true, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct LRUCacheOptions, num_shard_bits)}}, {"strict_capacity_limit", {offset_of(&LRUCacheOptions::strict_capacity_limit), - OptionType::kBoolean, OptionVerificationType::kNormal, true, + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kMutable, offsetof(struct LRUCacheOptions, strict_capacity_limit)}}, {"high_pri_pool_ratio", {offset_of(&LRUCacheOptions::high_pri_pool_ratio), OptionType::kDouble, - OptionVerificationType::kNormal, true, + OptionVerificationType::kNormal, OptionTypeFlags::kMutable, offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; #endif // !ROCKSDB_LITE diff --git a/options/options_helper.h b/options/options_helper.h index c75fdb1ce..6feca8f3d 100644 --- a/options/options_helper.h +++ b/options/options_helper.h @@ -12,6 +12,7 @@ #include "options/cf_options.h" #include "options/db_options.h" +#include "options/options_type.h" #include "rocksdb/options.h" #include "rocksdb/status.h" #include "rocksdb/table.h" @@ -28,6 +29,11 @@ ColumnFamilyOptions BuildColumnFamilyOptions( #ifndef ROCKSDB_LITE +Status ParseColumnFamilyOption(const std::string& name, + const std::string& org_value, + ColumnFamilyOptions* new_options, + bool input_strings_escaped = false); + Status GetMutableOptionsFromStrings( const MutableCFOptions& base_options, const std::unordered_map& options_map, @@ -44,71 +50,9 @@ Status GetTableFactoryFromMap( std::shared_ptr* table_factory, bool ignore_unknown_options = false); -enum class OptionType { - kBoolean, - kInt, - kInt32T, - kInt64T, - kVectorInt, - kUInt, - kUInt32T, - kUInt64T, - kSizeT, - kString, - kDouble, - kCompactionStyle, - kCompactionPri, - kSliceTransform, - kCompressionType, - kVectorCompressionType, - kTableFactory, - kComparator, - kCompactionFilter, - kCompactionFilterFactory, - kCompactionOptionsFIFO, - kCompactionOptionsUniversal, - kCompactionStopStyle, - kMergeOperator, - kMemTableRepFactory, - kBlockBasedTableIndexType, - kBlockBasedTableDataBlockIndexType, - kBlockBasedTableIndexShorteningMode, - kFilterPolicy, - kFlushBlockPolicyFactory, - kChecksumType, - kEncodingType, - kWALRecoveryMode, - kAccessHint, - kInfoLogLevel, - kLRUCacheOptions, - kEnv, - kUnknown, -}; - -enum class OptionVerificationType { - kNormal, - kByName, // The option is pointer typed so we can only verify - // based on it's name. - kByNameAllowNull, // Same as kByName, but it also allows the case - // where one of them is a nullptr. - kByNameAllowFromNull, // Same as kByName, but it also allows the case - // where the old option is nullptr. - kDeprecated // The option is no longer used in rocksdb. The RocksDB - // OptionsParser will still accept this option if it - // happen to exists in some Options file. However, - // the parser will not include it in serialization - // and verification processes. -}; - -// A struct for storing constant option information such as option name, -// option type, and offset. -struct OptionTypeInfo { - int offset; - OptionType type; - OptionVerificationType verification; - bool is_mutable; - int mutable_offset; -}; +Status ParseCompressionOptions(const std::string& value, + const std::string& name, + CompressionOptions& compression_opts); // A helper function that converts "opt_address" to a std::string // based on the specified OptionType. @@ -145,6 +89,10 @@ extern Status StringToMap( extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, const std::string& value); +Status GetStringFromStruct( + std::string* opt_string, const void* const options, + const std::unordered_map& type_info, + const std::string& delimiter); #endif // !ROCKSDB_LITE struct OptionsHelper { diff --git a/options/options_parser.cc b/options/options_parser.cc index 12be6a870..0f8eecc9a 100644 --- a/options/options_parser.cc +++ b/options/options_parser.cc @@ -620,10 +620,7 @@ bool AreEqualOptions( return false; } default: - if (type_info.verification == OptionVerificationType::kByName || - type_info.verification == - OptionVerificationType::kByNameAllowFromNull || - type_info.verification == OptionVerificationType::kByNameAllowNull) { + if (type_info.IsByName()) { std::string value1; bool result = SerializeSingleOptionHelper(offset1, type_info.type, &value1); @@ -637,13 +634,12 @@ bool AreEqualOptions( if (iter == opt_map->end()) { return true; } else { - if (type_info.verification == - OptionVerificationType::kByNameAllowNull) { + if (type_info.IsEnabled(OptionVerificationType::kByNameAllowNull)) { if (iter->second == kNullptrString || value1 == kNullptrString) { return true; } - } else if (type_info.verification == - OptionVerificationType::kByNameAllowFromNull) { + } else if (type_info.IsEnabled( + OptionVerificationType::kByNameAllowFromNull)) { if (iter->second == kNullptrString) { return true; } @@ -684,7 +680,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( // Verify ColumnFamily Name if (cf_names.size() != parser.cf_names()->size()) { - if (sanity_check_level >= kSanityLevelLooselyCompatible) { + if (sanity_check_level >= + OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) { return Status::InvalidArgument( "[RocksDBOptionParser Error] The persisted options does not have " "the same number of column family names as the db instance."); @@ -706,7 +703,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( // Verify Column Family Options if (cf_opts.size() != parser.cf_opts()->size()) { - if (sanity_check_level >= kSanityLevelLooselyCompatible) { + if (sanity_check_level >= + OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) { return Status::InvalidArgument( "[RocksDBOptionsParser Error]", "The persisted options does not have the same number of " @@ -740,7 +738,7 @@ Status RocksDBOptionsParser::VerifyDBOptions( const std::unordered_map* /*opt_map*/, OptionsSanityCheckLevel sanity_check_level) { for (const auto& pair : db_options_type_info) { - if (pair.second.verification == OptionVerificationType::kDeprecated) { + if (pair.second.IsDeprecated()) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; @@ -778,7 +776,7 @@ Status RocksDBOptionsParser::VerifyCFOptions( const std::unordered_map* persisted_opt_map, OptionsSanityCheckLevel sanity_check_level) { for (const auto& pair : cf_options_type_info) { - if (pair.second.verification == OptionVerificationType::kDeprecated) { + if (pair.second.IsDeprecated()) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; @@ -814,7 +812,7 @@ Status RocksDBOptionsParser::VerifyTableFactory( const TableFactory* base_tf, const TableFactory* file_tf, OptionsSanityCheckLevel sanity_check_level) { if (base_tf && file_tf) { - if (sanity_check_level > kSanityLevelNone && + if (sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone && std::string(base_tf->Name()) != std::string(file_tf->Name())) { return Status::Corruption( "[RocksDBOptionsParser]: " diff --git a/options/options_type.h b/options/options_type.h new file mode 100644 index 000000000..51e0c0031 --- /dev/null +++ b/options/options_type.h @@ -0,0 +1,141 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#include "rocksdb/rocksdb_namespace.h" + +namespace ROCKSDB_NAMESPACE { + +enum class OptionType { + kBoolean, + kInt, + kInt32T, + kInt64T, + kVectorInt, + kUInt, + kUInt32T, + kUInt64T, + kSizeT, + kString, + kDouble, + kCompactionStyle, + kCompactionPri, + kSliceTransform, + kCompressionType, + kVectorCompressionType, + kTableFactory, + kComparator, + kCompactionFilter, + kCompactionFilterFactory, + kCompactionOptionsFIFO, + kCompactionOptionsUniversal, + kCompactionStopStyle, + kMergeOperator, + kMemTableRepFactory, + kBlockBasedTableIndexType, + kBlockBasedTableDataBlockIndexType, + kBlockBasedTableIndexShorteningMode, + kFilterPolicy, + kFlushBlockPolicyFactory, + kChecksumType, + kEncodingType, + kWALRecoveryMode, + kAccessHint, + kInfoLogLevel, + kLRUCacheOptions, + kEnv, + kUnknown, +}; + +enum class OptionVerificationType { + kNormal, + kByName, // The option is pointer typed so we can only verify + // based on it's name. + kByNameAllowNull, // Same as kByName, but it also allows the case + // where one of them is a nullptr. + kByNameAllowFromNull, // Same as kByName, but it also allows the case + // where the old option is nullptr. + kDeprecated // The option is no longer used in rocksdb. The RocksDB + // OptionsParser will still accept this option if it + // happen to exists in some Options file. However, + // the parser will not include it in serialization + // and verification processes. +}; + +enum class OptionTypeFlags : uint32_t { + kNone = 0x00, // No flags + kMutable = 0x01, // Option is mutable +}; + +inline OptionTypeFlags operator|(const OptionTypeFlags &a, + const OptionTypeFlags &b) { + return static_cast(static_cast(a) | + static_cast(b)); +} + +inline OptionTypeFlags operator&(const OptionTypeFlags &a, + const OptionTypeFlags &b) { + return static_cast(static_cast(a) & + static_cast(b)); +} + +// A struct for storing constant option information such as option name, +// option type, and offset. +class OptionTypeInfo { + public: + int offset; + int mutable_offset; + OptionType type; + + // A simple "normal", non-mutable Type "_type" at _offset + OptionTypeInfo(int _offset, OptionType _type) + : offset(_offset), + mutable_offset(0), + type(_type), + verification(OptionVerificationType::kNormal), + flags(OptionTypeFlags::kNone) {} + + // A simple "normal", mutable Type "_type" at _offset + OptionTypeInfo(int _offset, OptionType _type, int _mutable_offset) + : offset(_offset), + mutable_offset(_mutable_offset), + type(_type), + verification(OptionVerificationType::kNormal), + flags(OptionTypeFlags::kMutable) {} + + OptionTypeInfo(int _offset, OptionType _type, + OptionVerificationType _verification, OptionTypeFlags _flags, + int _mutable_offset) + : offset(_offset), + mutable_offset(_mutable_offset), + type(_type), + verification(_verification), + flags(_flags) {} + + bool IsEnabled(OptionTypeFlags otf) const { return (flags & otf) == otf; } + + bool IsMutable() const { return IsEnabled(OptionTypeFlags::kMutable); } + + bool IsDeprecated() const { + return IsEnabled(OptionVerificationType::kDeprecated); + } + + bool IsEnabled(OptionVerificationType ovf) const { + return verification == ovf; + } + + bool IsByName() const { + return (verification == OptionVerificationType::kByName || + verification == OptionVerificationType::kByNameAllowNull || + verification == OptionVerificationType::kByNameAllowFromNull); + } + + protected: + private: + OptionVerificationType verification; + OptionTypeFlags flags; +}; +} // namespace ROCKSDB_NAMESPACE diff --git a/table/block_based/block_based_table_factory.cc b/table/block_based/block_based_table_factory.cc index 70a6f38d5..b79f8e7de 100644 --- a/table/block_based/block_based_table_factory.cc +++ b/table/block_based/block_based_table_factory.cc @@ -157,6 +157,126 @@ size_t TailPrefetchStats::GetSuggestedPrefetchSize() { return std::min(kMaxPrefetchSize, max_qualified_size); } +#ifndef ROCKSDB_LITE +static std::unordered_map + block_based_table_type_info = { + /* currently not supported + std::shared_ptr block_cache = nullptr; + std::shared_ptr block_cache_compressed = nullptr; + */ + {"flush_block_policy_factory", + {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), + OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"cache_index_and_filter_blocks", + {offsetof(struct BlockBasedTableOptions, + cache_index_and_filter_blocks), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"cache_index_and_filter_blocks_with_high_priority", + {offsetof(struct BlockBasedTableOptions, + cache_index_and_filter_blocks_with_high_priority), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"pin_l0_filter_and_index_blocks_in_cache", + {offsetof(struct BlockBasedTableOptions, + pin_l0_filter_and_index_blocks_in_cache), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"index_type", + {offsetof(struct BlockBasedTableOptions, index_type), + OptionType::kBlockBasedTableIndexType, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"hash_index_allow_collision", + {offsetof(struct BlockBasedTableOptions, hash_index_allow_collision), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"data_block_index_type", + {offsetof(struct BlockBasedTableOptions, data_block_index_type), + OptionType::kBlockBasedTableDataBlockIndexType, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"index_shortening", + {offsetof(struct BlockBasedTableOptions, index_shortening), + OptionType::kBlockBasedTableIndexShorteningMode, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"data_block_hash_table_util_ratio", + {offsetof(struct BlockBasedTableOptions, + data_block_hash_table_util_ratio), + OptionType::kDouble, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"checksum", + {offsetof(struct BlockBasedTableOptions, checksum), + OptionType::kChecksumType, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"no_block_cache", + {offsetof(struct BlockBasedTableOptions, no_block_cache), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"block_size", + {offsetof(struct BlockBasedTableOptions, block_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"block_size_deviation", + {offsetof(struct BlockBasedTableOptions, block_size_deviation), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"block_restart_interval", + {offsetof(struct BlockBasedTableOptions, block_restart_interval), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"index_block_restart_interval", + {offsetof(struct BlockBasedTableOptions, index_block_restart_interval), + OptionType::kInt, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"index_per_partition", + {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"metadata_block_size", + {offsetof(struct BlockBasedTableOptions, metadata_block_size), + OptionType::kUInt64T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"partition_filters", + {offsetof(struct BlockBasedTableOptions, partition_filters), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"filter_policy", + {offsetof(struct BlockBasedTableOptions, filter_policy), + OptionType::kFilterPolicy, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"whole_key_filtering", + {offsetof(struct BlockBasedTableOptions, whole_key_filtering), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"skip_table_builder_flush", + {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, + OptionTypeFlags::kNone, 0}}, + {"format_version", + {offsetof(struct BlockBasedTableOptions, format_version), + OptionType::kUInt32T, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"verify_compression", + {offsetof(struct BlockBasedTableOptions, verify_compression), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"read_amp_bytes_per_bit", + {offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"enable_index_compression", + {offsetof(struct BlockBasedTableOptions, enable_index_compression), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"block_align", + {offsetof(struct BlockBasedTableOptions, block_align), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"pin_top_level_index_and_filter", + {offsetof(struct BlockBasedTableOptions, + pin_top_level_index_and_filter), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}}; +#endif // ROCKSDB_LITE + // TODO(myabandeh): We should return an error instead of silently changing the // options BlockBasedTableFactory::BlockBasedTableFactory( @@ -416,46 +536,13 @@ std::string BlockBasedTableFactory::GetPrintableTableOptions() const { } #ifndef ROCKSDB_LITE -namespace { -bool SerializeSingleBlockBasedTableOption( - std::string* opt_string, const BlockBasedTableOptions& bbt_options, - const std::string& name, const std::string& delimiter) { - auto iter = block_based_table_type_info.find(name); - if (iter == block_based_table_type_info.end()) { - return false; - } - auto& opt_info = iter->second; - const char* opt_address = - reinterpret_cast(&bbt_options) + opt_info.offset; - std::string value; - bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value); - if (result) { - *opt_string = name + "=" + value + delimiter; - } - return result; -} -} // namespace Status BlockBasedTableFactory::GetOptionString( std::string* opt_string, const std::string& delimiter) const { assert(opt_string); opt_string->clear(); - for (auto iter = block_based_table_type_info.begin(); - iter != block_based_table_type_info.end(); ++iter) { - if (iter->second.verification == OptionVerificationType::kDeprecated) { - // If the option is no longer used in rocksdb and marked as deprecated, - // we skip it in the serialization. - continue; - } - std::string single_output; - bool result = SerializeSingleBlockBasedTableOption( - &single_output, table_options_, iter->first, delimiter); - assert(result); - if (result) { - opt_string->append(single_output); - } - } - return Status::OK(); + return GetStringFromStruct(opt_string, &table_options_, + block_based_table_type_info, delimiter); } #else Status BlockBasedTableFactory::GetOptionString( @@ -535,7 +622,7 @@ std::string ParseBlockBasedTableOption(const std::string& name, } } const auto& opt_info = iter->second; - if (opt_info.verification != OptionVerificationType::kDeprecated && + if (!opt_info.IsDeprecated() && !ParseOptionHelper(reinterpret_cast(new_options) + opt_info.offset, opt_info.type, value)) { return "Invalid value"; @@ -574,12 +661,7 @@ Status GetBlockBasedTableOptionsFromMap( !input_strings_escaped || // !input_strings_escaped indicates // the old API, where everything is // parsable. - (iter->second.verification != OptionVerificationType::kByName && - iter->second.verification != - OptionVerificationType::kByNameAllowNull && - iter->second.verification != - OptionVerificationType::kByNameAllowFromNull && - iter->second.verification != OptionVerificationType::kDeprecated)) { + (!iter->second.IsByName() && !iter->second.IsDeprecated())) { // Restore "new_options" to the default "base_options". *new_table_options = table_options; return Status::InvalidArgument("Can't parse BlockBasedTableOptions:", @@ -595,7 +677,7 @@ Status VerifyBlockBasedTableFactory( const BlockBasedTableFactory* file_tf, OptionsSanityCheckLevel sanity_check_level) { if ((base_tf != nullptr) != (file_tf != nullptr) && - sanity_check_level > kSanityLevelNone) { + sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone) { return Status::Corruption( "[RocksDBOptionsParser]: Inconsistent TableFactory class type"); } @@ -608,7 +690,7 @@ Status VerifyBlockBasedTableFactory( const auto& file_opt = file_tf->table_options(); for (auto& pair : block_based_table_type_info) { - if (pair.second.verification == OptionVerificationType::kDeprecated) { + if (pair.second.IsDeprecated()) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; diff --git a/table/block_based/block_based_table_factory.h b/table/block_based/block_based_table_factory.h index 7c8633c07..6bc01cee6 100644 --- a/table/block_based/block_based_table_factory.h +++ b/table/block_based/block_based_table_factory.h @@ -93,103 +93,5 @@ extern Status VerifyBlockBasedTableFactory( const BlockBasedTableFactory* file_tf, OptionsSanityCheckLevel sanity_check_level); -static std::unordered_map - block_based_table_type_info = { - /* currently not supported - std::shared_ptr block_cache = nullptr; - std::shared_ptr block_cache_compressed = nullptr; - */ - {"flush_block_policy_factory", - {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), - OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName, - false, 0}}, - {"cache_index_and_filter_blocks", - {offsetof(struct BlockBasedTableOptions, - cache_index_and_filter_blocks), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"cache_index_and_filter_blocks_with_high_priority", - {offsetof(struct BlockBasedTableOptions, - cache_index_and_filter_blocks_with_high_priority), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"pin_l0_filter_and_index_blocks_in_cache", - {offsetof(struct BlockBasedTableOptions, - pin_l0_filter_and_index_blocks_in_cache), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"index_type", - {offsetof(struct BlockBasedTableOptions, index_type), - OptionType::kBlockBasedTableIndexType, - OptionVerificationType::kNormal, false, 0}}, - {"hash_index_allow_collision", - {offsetof(struct BlockBasedTableOptions, hash_index_allow_collision), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"data_block_index_type", - {offsetof(struct BlockBasedTableOptions, data_block_index_type), - OptionType::kBlockBasedTableDataBlockIndexType, - OptionVerificationType::kNormal, false, 0}}, - {"index_shortening", - {offsetof(struct BlockBasedTableOptions, index_shortening), - OptionType::kBlockBasedTableIndexShorteningMode, - OptionVerificationType::kNormal, false, 0}}, - {"data_block_hash_table_util_ratio", - {offsetof(struct BlockBasedTableOptions, - data_block_hash_table_util_ratio), - OptionType::kDouble, OptionVerificationType::kNormal, false, 0}}, - {"checksum", - {offsetof(struct BlockBasedTableOptions, checksum), - OptionType::kChecksumType, OptionVerificationType::kNormal, false, - 0}}, - {"no_block_cache", - {offsetof(struct BlockBasedTableOptions, no_block_cache), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"block_size", - {offsetof(struct BlockBasedTableOptions, block_size), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"block_size_deviation", - {offsetof(struct BlockBasedTableOptions, block_size_deviation), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"block_restart_interval", - {offsetof(struct BlockBasedTableOptions, block_restart_interval), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"index_block_restart_interval", - {offsetof(struct BlockBasedTableOptions, index_block_restart_interval), - OptionType::kInt, OptionVerificationType::kNormal, false, 0}}, - {"index_per_partition", - {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, false, - 0}}, - {"metadata_block_size", - {offsetof(struct BlockBasedTableOptions, metadata_block_size), - OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, - {"partition_filters", - {offsetof(struct BlockBasedTableOptions, partition_filters), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"filter_policy", - {offsetof(struct BlockBasedTableOptions, filter_policy), - OptionType::kFilterPolicy, OptionVerificationType::kByName, false, - 0}}, - {"whole_key_filtering", - {offsetof(struct BlockBasedTableOptions, whole_key_filtering), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"skip_table_builder_flush", - {0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false, - 0}}, - {"format_version", - {offsetof(struct BlockBasedTableOptions, format_version), - OptionType::kUInt32T, OptionVerificationType::kNormal, false, 0}}, - {"verify_compression", - {offsetof(struct BlockBasedTableOptions, verify_compression), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"read_amp_bytes_per_bit", - {offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"enable_index_compression", - {offsetof(struct BlockBasedTableOptions, enable_index_compression), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"block_align", - {offsetof(struct BlockBasedTableOptions, block_align), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}, - {"pin_top_level_index_and_filter", - {offsetof(struct BlockBasedTableOptions, - pin_top_level_index_and_filter), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}}; #endif // !ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/table/plain/plain_table_factory.cc b/table/plain/plain_table_factory.cc index c2db8f395..df296cdff 100644 --- a/table/plain/plain_table_factory.cc +++ b/table/plain/plain_table_factory.cc @@ -17,6 +17,34 @@ #include "util/string_util.h" namespace ROCKSDB_NAMESPACE { +static std::unordered_map plain_table_type_info = { + {"user_key_len", + {offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"bloom_bits_per_key", + {offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"hash_table_ratio", + {offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"index_sparseness", + {offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"huge_page_tlb_size", + {offsetof(struct PlainTableOptions, huge_page_tlb_size), + OptionType::kSizeT, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}, + {"encoding_type", + {offsetof(struct PlainTableOptions, encoding_type), + OptionType::kEncodingType, OptionVerificationType::kByName, + OptionTypeFlags::kNone, 0}}, + {"full_scan_mode", + {offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean, + OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, + {"store_index_in_file", + {offsetof(struct PlainTableOptions, store_index_in_file), + OptionType::kBoolean, OptionVerificationType::kNormal, + OptionTypeFlags::kNone, 0}}}; Status PlainTableFactory::NewTableReader( const TableReaderOptions& table_reader_options, @@ -178,7 +206,7 @@ std::string ParsePlainTableOptions(const std::string& name, } } const auto& opt_info = iter->second; - if (opt_info.verification != OptionVerificationType::kDeprecated && + if (!opt_info.IsDeprecated() && !ParseOptionHelper(reinterpret_cast(new_options) + opt_info.offset, opt_info.type, value)) { return "Invalid value"; @@ -202,12 +230,7 @@ Status GetPlainTableOptionsFromMap( !input_strings_escaped || // !input_strings_escaped indicates // the old API, where everything is // parsable. - (iter->second.verification != OptionVerificationType::kByName && - iter->second.verification != - OptionVerificationType::kByNameAllowNull && - iter->second.verification != - OptionVerificationType::kByNameAllowFromNull && - iter->second.verification != OptionVerificationType::kDeprecated)) { + (!iter->second.IsByName() && !iter->second.IsDeprecated())) { // Restore "new_options" to the default "base_options". *new_table_options = table_options; return Status::InvalidArgument("Can't parse PlainTableOptions:", diff --git a/table/plain/plain_table_factory.h b/table/plain/plain_table_factory.h index 64dd171cb..cbd892a9e 100644 --- a/table/plain/plain_table_factory.h +++ b/table/plain/plain_table_factory.h @@ -193,31 +193,6 @@ class PlainTableFactory : public TableFactory { PlainTableOptions table_options_; }; -static std::unordered_map plain_table_type_info = { - {"user_key_len", - {offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T, - OptionVerificationType::kNormal, false, 0}}, - {"bloom_bits_per_key", - {offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt, - OptionVerificationType::kNormal, false, 0}}, - {"hash_table_ratio", - {offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble, - OptionVerificationType::kNormal, false, 0}}, - {"index_sparseness", - {offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT, - OptionVerificationType::kNormal, false, 0}}, - {"huge_page_tlb_size", - {offsetof(struct PlainTableOptions, huge_page_tlb_size), - OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}}, - {"encoding_type", - {offsetof(struct PlainTableOptions, encoding_type), - OptionType::kEncodingType, OptionVerificationType::kByName, false, 0}}, - {"full_scan_mode", - {offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean, - OptionVerificationType::kNormal, false, 0}}, - {"store_index_in_file", - {offsetof(struct PlainTableOptions, store_index_in_file), - OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}}; } // namespace ROCKSDB_NAMESPACE #endif // ROCKSDB_LITE