Better handling of deprecated options in RocksDBOptionsParser

Summary:
Previously, we treat deprecated options as normal options in
RocksDBOptionsParser.  However, these deprecated options should
not be verified and serialized.

Test Plan: options_test

Reviewers: igor, sdong, IslamAbdelRahman, anthony

Reviewed By: anthony

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D47775
main
Yueh-Hsuan Chiang 9 years ago
parent a8b8295d18
commit 1e73b11af7
  1. 3
      util/options_helper.cc
  2. 208
      util/options_helper.h
  3. 10
      util/options_parser.cc

@ -702,6 +702,9 @@ Status GetStringFromColumnFamilyOptions(std::string* opt_string,
opt_string->clear();
for (auto iter = cf_options_type_info.begin();
iter != cf_options_type_info.end(); ++iter) {
if (iter->second.verification == OptionVerificationType::kDeprecated) {
continue;
}
std::string single_output;
bool result = SerializeSingleColumnFamilyOption(&single_output, cf_options,
iter->first, delimiter);

@ -68,11 +68,21 @@ enum class OptionType {
kUnknown
};
enum class OptionVerificationType {
kNormal,
kDeprecated // The option is no longer used in rocksdb. The RocksDB
// OptionsParser will still accept this option if it
// happen to exists in some Options file. However, the
// parser will not include it in serialization and
// verification processes.
};
// A struct for storing constant option information such as option name,
// option type, and offset.
struct OptionTypeInfo {
int offset;
OptionType type;
OptionVerificationType verification;
};
static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
@ -91,94 +101,125 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
std::vector<std::shared_ptr<EventListener>> listeners;
*/
{"advise_random_on_open",
{offsetof(struct DBOptions, advise_random_on_open), OptionType::kBoolean}},
{offsetof(struct DBOptions, advise_random_on_open), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"allow_mmap_reads",
{offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean}},
{offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"allow_mmap_writes",
{offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean}},
{offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"allow_os_buffer",
{offsetof(struct DBOptions, allow_os_buffer), OptionType::kBoolean}},
{offsetof(struct DBOptions, allow_os_buffer), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"create_if_missing",
{offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean}},
{offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"create_missing_column_families",
{offsetof(struct DBOptions, create_missing_column_families),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"disableDataSync",
{offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}},
{offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"disable_data_sync", // for compatibility
{offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}},
{offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"enable_thread_tracking",
{offsetof(struct DBOptions, enable_thread_tracking),
OptionType::kBoolean}},
{offsetof(struct DBOptions, enable_thread_tracking), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"error_if_exists",
{offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean}},
{offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"is_fd_close_on_exec",
{offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean}},
{offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"paranoid_checks",
{offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean}},
{offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"skip_log_error_on_recovery",
{offsetof(struct DBOptions, skip_log_error_on_recovery),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"skip_stats_update_on_db_open",
{offsetof(struct DBOptions, skip_stats_update_on_db_open),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"new_table_reader_for_compaction_inputs",
{offsetof(struct DBOptions, new_table_reader_for_compaction_inputs),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"compaction_readahead_size",
{offsetof(struct DBOptions, compaction_readahead_size),
OptionType::kSizeT}},
{offsetof(struct DBOptions, compaction_readahead_size), OptionType::kSizeT,
OptionVerificationType::kNormal}},
{"use_adaptive_mutex",
{offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean}},
{offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"use_fsync",
{offsetof(struct DBOptions, use_fsync), OptionType::kBoolean}},
{offsetof(struct DBOptions, use_fsync), OptionType::kBoolean,
OptionVerificationType::kNormal}},
{"max_background_compactions",
{offsetof(struct DBOptions, max_background_compactions),
OptionType::kInt}},
{offsetof(struct DBOptions, max_background_compactions), OptionType::kInt,
OptionVerificationType::kNormal}},
{"max_background_flushes",
{offsetof(struct DBOptions, max_background_flushes), OptionType::kInt}},
{offsetof(struct DBOptions, max_background_flushes), OptionType::kInt,
OptionVerificationType::kNormal}},
{"max_file_opening_threads",
{offsetof(struct DBOptions, max_file_opening_threads), OptionType::kInt}},
{offsetof(struct DBOptions, max_file_opening_threads), OptionType::kInt,
OptionVerificationType::kNormal}},
{"max_open_files",
{offsetof(struct DBOptions, max_open_files), OptionType::kInt}},
{offsetof(struct DBOptions, max_open_files), OptionType::kInt,
OptionVerificationType::kNormal}},
{"table_cache_numshardbits",
{offsetof(struct DBOptions, table_cache_numshardbits), OptionType::kInt}},
{offsetof(struct DBOptions, table_cache_numshardbits), OptionType::kInt,
OptionVerificationType::kNormal}},
{"db_write_buffer_size",
{offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT}},
{offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT,
OptionVerificationType::kNormal}},
{"keep_log_file_num",
{offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT}},
{offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT,
OptionVerificationType::kNormal}},
{"log_file_time_to_roll",
{offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT}},
{offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT,
OptionVerificationType::kNormal}},
{"manifest_preallocation_size",
{offsetof(struct DBOptions, manifest_preallocation_size),
OptionType::kSizeT}},
OptionType::kSizeT, OptionVerificationType::kNormal}},
{"max_log_file_size",
{offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT}},
{offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT,
OptionVerificationType::kNormal}},
{"db_log_dir",
{offsetof(struct DBOptions, db_log_dir), OptionType::kString}},
{"wal_dir", {offsetof(struct DBOptions, wal_dir), OptionType::kString}},
{offsetof(struct DBOptions, db_log_dir), OptionType::kString,
OptionVerificationType::kNormal}},
{"wal_dir",
{offsetof(struct DBOptions, wal_dir), OptionType::kString,
OptionVerificationType::kNormal}},
{"max_subcompactions",
{offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T}},
{offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T,
OptionVerificationType::kNormal}},
{"WAL_size_limit_MB",
{offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T}},
{offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T,
OptionVerificationType::kNormal}},
{"WAL_ttl_seconds",
{offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T}},
{offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T,
OptionVerificationType::kNormal}},
{"bytes_per_sync",
{offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T}},
{offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T,
OptionVerificationType::kNormal}},
{"delayed_write_rate",
{offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T}},
{offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T,
OptionVerificationType::kNormal}},
{"delete_obsolete_files_period_micros",
{offsetof(struct DBOptions, delete_obsolete_files_period_micros),
OptionType::kUInt64T}},
OptionType::kUInt64T, OptionVerificationType::kNormal}},
{"max_manifest_file_size",
{offsetof(struct DBOptions, max_manifest_file_size),
OptionType::kUInt64T}},
{offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T,
OptionVerificationType::kNormal}},
{"max_total_wal_size",
{offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T}},
{offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T,
OptionVerificationType::kNormal}},
{"wal_bytes_per_sync",
{offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T}},
{offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T,
OptionVerificationType::kNormal}},
{"stats_dump_period_sec",
{offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt}}};
{offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt,
OptionVerificationType::kNormal}}};
static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
/* not yet supported
@ -205,122 +246,123 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
*/
{"compaction_measure_io_stats",
{offsetof(struct ColumnFamilyOptions, compaction_measure_io_stats),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"disable_auto_compactions",
{offsetof(struct ColumnFamilyOptions, disable_auto_compactions),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"filter_deletes",
{offsetof(struct ColumnFamilyOptions, filter_deletes),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"inplace_update_support",
{offsetof(struct ColumnFamilyOptions, inplace_update_support),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"level_compaction_dynamic_level_bytes",
{offsetof(struct ColumnFamilyOptions,
level_compaction_dynamic_level_bytes),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"optimize_filters_for_hits",
{offsetof(struct ColumnFamilyOptions, optimize_filters_for_hits),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"paranoid_file_checks",
{offsetof(struct ColumnFamilyOptions, paranoid_file_checks),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"purge_redundant_kvs_while_flush",
{offsetof(struct ColumnFamilyOptions, purge_redundant_kvs_while_flush),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"verify_checksums_in_compaction",
{offsetof(struct ColumnFamilyOptions, verify_checksums_in_compaction),
OptionType::kBoolean}},
OptionType::kBoolean, OptionVerificationType::kNormal}},
{"hard_pending_compaction_bytes_limit",
{offsetof(struct ColumnFamilyOptions, hard_pending_compaction_bytes_limit),
OptionType::kUInt64T}},
OptionType::kUInt64T, OptionVerificationType::kNormal}},
{"hard_rate_limit",
{offsetof(struct ColumnFamilyOptions, hard_rate_limit),
OptionType::kDouble}},
OptionType::kDouble, OptionVerificationType::kDeprecated}},
{"soft_rate_limit",
{offsetof(struct ColumnFamilyOptions, soft_rate_limit),
OptionType::kDouble}},
OptionType::kDouble, OptionVerificationType::kNormal}},
{"expanded_compaction_factor",
{offsetof(struct ColumnFamilyOptions, expanded_compaction_factor),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"level0_file_num_compaction_trigger",
{offsetof(struct ColumnFamilyOptions, level0_file_num_compaction_trigger),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"level0_slowdown_writes_trigger",
{offsetof(struct ColumnFamilyOptions, level0_slowdown_writes_trigger),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"level0_stop_writes_trigger",
{offsetof(struct ColumnFamilyOptions, level0_stop_writes_trigger),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"max_bytes_for_level_multiplier",
{offsetof(struct ColumnFamilyOptions, max_bytes_for_level_multiplier),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"max_grandparent_overlap_factor",
{offsetof(struct ColumnFamilyOptions, max_grandparent_overlap_factor),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"max_mem_compaction_level",
{offsetof(struct ColumnFamilyOptions, max_mem_compaction_level),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kDeprecated}},
{"max_write_buffer_number",
{offsetof(struct ColumnFamilyOptions, max_write_buffer_number),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"max_write_buffer_number_to_maintain",
{offsetof(struct ColumnFamilyOptions, max_write_buffer_number_to_maintain),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"min_write_buffer_number_to_merge",
{offsetof(struct ColumnFamilyOptions, min_write_buffer_number_to_merge),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"num_levels",
{offsetof(struct ColumnFamilyOptions, num_levels), OptionType::kInt}},
{offsetof(struct ColumnFamilyOptions, num_levels), OptionType::kInt,
OptionVerificationType::kNormal}},
{"source_compaction_factor",
{offsetof(struct ColumnFamilyOptions, source_compaction_factor),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"target_file_size_multiplier",
{offsetof(struct ColumnFamilyOptions, target_file_size_multiplier),
OptionType::kInt}},
OptionType::kInt, OptionVerificationType::kNormal}},
{"arena_block_size",
{offsetof(struct ColumnFamilyOptions, arena_block_size),
OptionType::kSizeT}},
OptionType::kSizeT, OptionVerificationType::kNormal}},
{"inplace_update_num_locks",
{offsetof(struct ColumnFamilyOptions, inplace_update_num_locks),
OptionType::kSizeT}},
OptionType::kSizeT, OptionVerificationType::kNormal}},
{"max_successive_merges",
{offsetof(struct ColumnFamilyOptions, max_successive_merges),
OptionType::kSizeT}},
OptionType::kSizeT, OptionVerificationType::kNormal}},
{"memtable_prefix_bloom_huge_page_tlb_size",
{offsetof(struct ColumnFamilyOptions,
memtable_prefix_bloom_huge_page_tlb_size),
OptionType::kSizeT}},
OptionType::kSizeT, OptionVerificationType::kNormal}},
{"write_buffer_size",
{offsetof(struct ColumnFamilyOptions, write_buffer_size),
OptionType::kSizeT}},
OptionType::kSizeT, OptionVerificationType::kNormal}},
{"bloom_locality",
{offsetof(struct ColumnFamilyOptions, bloom_locality),
OptionType::kUInt32T}},
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{"memtable_prefix_bloom_bits",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_bits),
OptionType::kUInt32T}},
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{"memtable_prefix_bloom_probes",
{offsetof(struct ColumnFamilyOptions, memtable_prefix_bloom_probes),
OptionType::kUInt32T}},
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{"min_partial_merge_operands",
{offsetof(struct ColumnFamilyOptions, min_partial_merge_operands),
OptionType::kUInt32T}},
OptionType::kUInt32T, OptionVerificationType::kNormal}},
{"max_bytes_for_level_base",
{offsetof(struct ColumnFamilyOptions, max_bytes_for_level_base),
OptionType::kUInt64T}},
OptionType::kUInt64T, OptionVerificationType::kNormal}},
{"max_sequential_skip_in_iterations",
{offsetof(struct ColumnFamilyOptions, max_sequential_skip_in_iterations),
OptionType::kUInt64T}},
OptionType::kUInt64T, OptionVerificationType::kNormal}},
{"target_file_size_base",
{offsetof(struct ColumnFamilyOptions, target_file_size_base),
OptionType::kUInt64T}},
OptionType::kUInt64T, OptionVerificationType::kNormal}},
{"rate_limit_delay_max_milliseconds",
{offsetof(struct ColumnFamilyOptions, rate_limit_delay_max_milliseconds),
OptionType::kUInt}},
OptionType::kUInt, OptionVerificationType::kDeprecated}},
{"compaction_style",
{offsetof(struct ColumnFamilyOptions, compaction_style),
OptionType::kCompactionStyle}}};
OptionType::kCompactionStyle, OptionVerificationType::kNormal}}};
} // namespace rocksdb

@ -532,6 +532,11 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
const DBOptions& new_opt) {
for (auto pair : db_options_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
}
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&new_opt),
pair.second)) {
@ -547,6 +552,11 @@ Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
Status RocksDBOptionsParser::VerifyCFOptions(
const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt) {
for (auto& pair : cf_options_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
}
if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
reinterpret_cast<const char*>(&new_opt),
pair.second)) {

Loading…
Cancel
Save