Move the OptionTypeMap code closer to home (#6198)

Summary:
This is a predecessor to the Configurable PR.  This change moves the OptionTypeInfo maps closer to where they will be used.

When the Configurable changes are adopted, these values will become static and not associated with the OptionsHelper.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6198

Reviewed By: siying

Differential Revision: D20778108

Pulled By: zhichao-cao

fbshipit-source-id: a9f85fc73bc53503656e1958ecc1e764052fd1aa
main
mrambacher 4 years ago committed by Facebook GitHub Bot
parent 079e77ff9e
commit 259b6ec8da
  1. 6
      db/db_options_test.cc
  2. 437
      options/cf_options.cc
  3. 316
      options/db_options.cc
  4. 842
      options/options_helper.cc
  5. 78
      options/options_helper.h
  6. 24
      options/options_parser.cc
  7. 141
      options/options_type.h
  8. 170
      table/block_based/block_based_table_factory.cc
  9. 98
      table/block_based/block_based_table_factory.h
  10. 37
      table/plain/plain_table_factory.cc
  11. 25
      table/plain/plain_table_factory.h

@ -38,8 +38,7 @@ class DBOptionsTest : public DBTestBase {
StringToMap(options_str, &options_map);
std::unordered_map<std::string, std::string> mutable_map;
for (const auto opt : db_options_type_info) {
if (opt.second.is_mutable &&
opt.second.verification != OptionVerificationType::kDeprecated) {
if (opt.second.IsMutable() && !opt.second.IsDeprecated()) {
mutable_map[opt.first] = options_map[opt.first];
}
}
@ -54,8 +53,7 @@ class DBOptionsTest : public DBTestBase {
StringToMap(options_str, &options_map);
std::unordered_map<std::string, std::string> mutable_map;
for (const auto opt : cf_options_type_info) {
if (opt.second.is_mutable &&
opt.second.verification != OptionVerificationType::kDeprecated) {
if (opt.second.IsMutable() && !opt.second.IsDeprecated()) {
mutable_map[opt.first] = options_map[opt.first];
}
}

@ -9,14 +9,451 @@
#include <cinttypes>
#include <limits>
#include <string>
#include "options/db_options.h"
#include "options/options_helper.h"
#include "port/port.h"
#include "rocksdb/concurrent_task_limiter.h"
#include "rocksdb/convenience.h"
#include "rocksdb/env.h"
#include "rocksdb/file_system.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/options.h"
#include "rocksdb/utilities/object_registry.h"
#include "table/block_based/block_based_table_factory.h"
#include "table/plain/plain_table_factory.h"
#include "util/cast_util.h"
namespace ROCKSDB_NAMESPACE {
// offset_of is used to get the offset of a class data member
// ex: offset_of(&ColumnFamilyOptions::num_levels)
// This call will return the offset of num_levels in ColumnFamilyOptions class
//
// This is the same as offsetof() but allow us to work with non standard-layout
// classes and structures
// refs:
// http://en.cppreference.com/w/cpp/concept/StandardLayoutType
// https://gist.github.com/graphitemaster/494f21190bb2c63c5516
#ifndef ROCKSDB_LITE
ColumnFamilyOptions OptionsHelper::dummy_cf_options;
template <typename T1>
int offset_of(T1 ColumnFamilyOptions::*member) {
return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) -
size_t(&OptionsHelper::dummy_cf_options));
}
template <typename T1>
int offset_of(T1 AdvancedColumnFamilyOptions::*member) {
return int(size_t(&(OptionsHelper::dummy_cf_options.*member)) -
size_t(&OptionsHelper::dummy_cf_options));
}
const std::string kNameComparator = "comparator";
const std::string kNameMergeOperator = "merge_operator";
std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::cf_options_type_info = {
/* not yet supported
CompressionOptions compression_opts;
TablePropertiesCollectorFactories table_properties_collector_factories;
typedef std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
TablePropertiesCollectorFactories;
UpdateStatus (*inplace_callback)(char* existing_value,
uint34_t* existing_value_size,
Slice delta_value,
std::string* merged_value);
std::vector<DbPath> cf_paths;
*/
{"report_bg_io_stats",
{offset_of(&ColumnFamilyOptions::report_bg_io_stats),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, report_bg_io_stats)}},
{"compaction_measure_io_stats",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"disable_auto_compactions",
{offset_of(&ColumnFamilyOptions::disable_auto_compactions),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, disable_auto_compactions)}},
{"filter_deletes",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"inplace_update_support",
{offset_of(&ColumnFamilyOptions::inplace_update_support),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"level_compaction_dynamic_level_bytes",
{offset_of(&ColumnFamilyOptions::level_compaction_dynamic_level_bytes),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"optimize_filters_for_hits",
{offset_of(&ColumnFamilyOptions::optimize_filters_for_hits),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"paranoid_file_checks",
{offset_of(&ColumnFamilyOptions::paranoid_file_checks),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, paranoid_file_checks)}},
{"force_consistency_checks",
{offset_of(&ColumnFamilyOptions::force_consistency_checks),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"purge_redundant_kvs_while_flush",
{offset_of(&ColumnFamilyOptions::purge_redundant_kvs_while_flush),
OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"verify_checksums_in_compaction",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"soft_pending_compaction_bytes_limit",
{offset_of(&ColumnFamilyOptions::soft_pending_compaction_bytes_limit),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions,
soft_pending_compaction_bytes_limit)}},
{"hard_pending_compaction_bytes_limit",
{offset_of(&ColumnFamilyOptions::hard_pending_compaction_bytes_limit),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions,
hard_pending_compaction_bytes_limit)}},
{"hard_rate_limit",
{0, OptionType::kDouble, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"soft_rate_limit",
{0, OptionType::kDouble, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"max_compaction_bytes",
{offset_of(&ColumnFamilyOptions::max_compaction_bytes),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, max_compaction_bytes)}},
{"expanded_compaction_factor",
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"level0_file_num_compaction_trigger",
{offset_of(&ColumnFamilyOptions::level0_file_num_compaction_trigger),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions,
level0_file_num_compaction_trigger)}},
{"level0_slowdown_writes_trigger",
{offset_of(&ColumnFamilyOptions::level0_slowdown_writes_trigger),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger)}},
{"level0_stop_writes_trigger",
{offset_of(&ColumnFamilyOptions::level0_stop_writes_trigger),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, level0_stop_writes_trigger)}},
{"max_grandparent_overlap_factor",
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"max_mem_compaction_level",
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"max_write_buffer_number",
{offset_of(&ColumnFamilyOptions::max_write_buffer_number),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, max_write_buffer_number)}},
{"max_write_buffer_number_to_maintain",
{offset_of(&ColumnFamilyOptions::max_write_buffer_number_to_maintain),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"max_write_buffer_size_to_maintain",
{offset_of(&ColumnFamilyOptions::max_write_buffer_size_to_maintain),
OptionType::kInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"min_write_buffer_number_to_merge",
{offset_of(&ColumnFamilyOptions::min_write_buffer_number_to_merge),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"num_levels",
{offset_of(&ColumnFamilyOptions::num_levels), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"source_compaction_factor",
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"target_file_size_multiplier",
{offset_of(&ColumnFamilyOptions::target_file_size_multiplier),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, target_file_size_multiplier)}},
{"arena_block_size",
{offset_of(&ColumnFamilyOptions::arena_block_size), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, arena_block_size)}},
{"inplace_update_num_locks",
{offset_of(&ColumnFamilyOptions::inplace_update_num_locks),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, inplace_update_num_locks)}},
{"max_successive_merges",
{offset_of(&ColumnFamilyOptions::max_successive_merges),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, max_successive_merges)}},
{"memtable_huge_page_size",
{offset_of(&ColumnFamilyOptions::memtable_huge_page_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, memtable_huge_page_size)}},
{"memtable_prefix_bloom_huge_page_tlb_size",
{0, OptionType::kSizeT, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"write_buffer_size",
{offset_of(&ColumnFamilyOptions::write_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, write_buffer_size)}},
{"bloom_locality",
{offset_of(&ColumnFamilyOptions::bloom_locality), OptionType::kUInt32T,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"memtable_prefix_bloom_bits",
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"memtable_prefix_bloom_size_ratio",
{offset_of(&ColumnFamilyOptions::memtable_prefix_bloom_size_ratio),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio)}},
{"memtable_prefix_bloom_probes",
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"memtable_whole_key_filtering",
{offset_of(&ColumnFamilyOptions::memtable_whole_key_filtering),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, memtable_whole_key_filtering)}},
{"min_partial_merge_operands",
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"max_bytes_for_level_base",
{offset_of(&ColumnFamilyOptions::max_bytes_for_level_base),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, max_bytes_for_level_base)}},
{"snap_refresh_nanos",
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"max_bytes_for_level_multiplier",
{offset_of(&ColumnFamilyOptions::max_bytes_for_level_multiplier),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}},
{"max_bytes_for_level_multiplier_additional",
{offset_of(
&ColumnFamilyOptions::max_bytes_for_level_multiplier_additional),
OptionType::kVectorInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions,
max_bytes_for_level_multiplier_additional)}},
{"max_sequential_skip_in_iterations",
{offset_of(&ColumnFamilyOptions::max_sequential_skip_in_iterations),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions,
max_sequential_skip_in_iterations)}},
{"target_file_size_base",
{offset_of(&ColumnFamilyOptions::target_file_size_base),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, target_file_size_base)}},
{"rate_limit_delay_max_milliseconds",
{0, OptionType::kUInt, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"compression",
{offset_of(&ColumnFamilyOptions::compression),
OptionType::kCompressionType, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, compression)}},
{"compression_per_level",
{offset_of(&ColumnFamilyOptions::compression_per_level),
OptionType::kVectorCompressionType, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"bottommost_compression",
{offset_of(&ColumnFamilyOptions::bottommost_compression),
OptionType::kCompressionType, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, bottommost_compression)}},
{kNameComparator,
{offset_of(&ColumnFamilyOptions::comparator), OptionType::kComparator,
OptionVerificationType::kByName, OptionTypeFlags::kNone, 0}},
{"prefix_extractor",
{offset_of(&ColumnFamilyOptions::prefix_extractor),
OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, prefix_extractor)}},
{"memtable_insert_with_hint_prefix_extractor",
{offset_of(
&ColumnFamilyOptions::memtable_insert_with_hint_prefix_extractor),
OptionType::kSliceTransform, OptionVerificationType::kByNameAllowNull,
OptionTypeFlags::kNone, 0}},
{"memtable_factory",
{offset_of(&ColumnFamilyOptions::memtable_factory),
OptionType::kMemTableRepFactory, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
{"table_factory",
{offset_of(&ColumnFamilyOptions::table_factory),
OptionType::kTableFactory, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
{"compaction_filter",
{offset_of(&ColumnFamilyOptions::compaction_filter),
OptionType::kCompactionFilter, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
{"compaction_filter_factory",
{offset_of(&ColumnFamilyOptions::compaction_filter_factory),
OptionType::kCompactionFilterFactory, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
{kNameMergeOperator,
{offset_of(&ColumnFamilyOptions::merge_operator),
OptionType::kMergeOperator,
OptionVerificationType::kByNameAllowFromNull, OptionTypeFlags::kNone,
0}},
{"compaction_style",
{offset_of(&ColumnFamilyOptions::compaction_style),
OptionType::kCompactionStyle, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"compaction_pri",
{offset_of(&ColumnFamilyOptions::compaction_pri),
OptionType::kCompactionPri, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"compaction_options_fifo",
{offset_of(&ColumnFamilyOptions::compaction_options_fifo),
OptionType::kCompactionOptionsFIFO, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, compaction_options_fifo)}},
{"compaction_options_universal",
{offset_of(&ColumnFamilyOptions::compaction_options_universal),
OptionType::kCompactionOptionsUniversal,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, compaction_options_universal)}},
{"ttl",
{offset_of(&ColumnFamilyOptions::ttl), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, ttl)}},
{"periodic_compaction_seconds",
{offset_of(&ColumnFamilyOptions::periodic_compaction_seconds),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, periodic_compaction_seconds)}},
{"sample_for_compression",
{offset_of(&ColumnFamilyOptions::sample_for_compression),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableCFOptions, sample_for_compression)}}};
Status ParseColumnFamilyOption(const std::string& name,
const std::string& org_value,
ColumnFamilyOptions* new_options,
bool input_strings_escaped) {
const std::string& value =
input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
try {
if (name == "block_based_table_factory") {
// Nested options
BlockBasedTableOptions table_opt, base_table_options;
BlockBasedTableFactory* block_based_table_factory =
static_cast_with_check<BlockBasedTableFactory, TableFactory>(
new_options->table_factory.get());
if (block_based_table_factory != nullptr) {
base_table_options = block_based_table_factory->table_options();
}
Status table_opt_s = GetBlockBasedTableOptionsFromString(
base_table_options, value, &table_opt);
if (!table_opt_s.ok()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt));
} else if (name == "plain_table_factory") {
// Nested options
PlainTableOptions table_opt, base_table_options;
PlainTableFactory* plain_table_factory =
static_cast_with_check<PlainTableFactory, TableFactory>(
new_options->table_factory.get());
if (plain_table_factory != nullptr) {
base_table_options = plain_table_factory->table_options();
}
Status table_opt_s =
GetPlainTableOptionsFromString(base_table_options, value, &table_opt);
if (!table_opt_s.ok()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
new_options->table_factory.reset(NewPlainTableFactory(table_opt));
} else if (name == "memtable") {
std::unique_ptr<MemTableRepFactory> new_mem_factory;
Status mem_factory_s =
GetMemTableRepFactoryFromString(value, &new_mem_factory);
if (!mem_factory_s.ok()) {
return Status::InvalidArgument(
"unable to parse the specified CF option " + name);
}
new_options->memtable_factory.reset(new_mem_factory.release());
} else if (name == "bottommost_compression_opts") {
Status s = ParseCompressionOptions(
value, name, new_options->bottommost_compression_opts);
if (!s.ok()) {
return s;
}
} else if (name == "compression_opts") {
Status s =
ParseCompressionOptions(value, name, new_options->compression_opts);
if (!s.ok()) {
return s;
}
} else {
if (name == kNameComparator) {
// Try to get comparator from object registry first.
// Only support static comparator for now.
Status status = ObjectRegistry::NewInstance()->NewStaticObject(
value, &new_options->comparator);
if (status.ok()) {
return status;
}
} else if (name == kNameMergeOperator) {
// Try to get merge operator from object registry first.
std::shared_ptr<MergeOperator> mo;
Status status =
ObjectRegistry::NewInstance()->NewSharedObject<MergeOperator>(
value, &new_options->merge_operator);
// Only support static comparator for now.
if (status.ok()) {
return status;
}
}
auto iter = cf_options_type_info.find(name);
if (iter == cf_options_type_info.end()) {
return Status::InvalidArgument(
"Unable to parse the specified CF option " + name);
}
const auto& opt_info = iter->second;
if (opt_info.IsDeprecated() ||
ParseOptionHelper(
reinterpret_cast<char*>(new_options) + opt_info.offset,
opt_info.type, value)) {
return Status::OK();
} else if (opt_info.IsByName()) {
return Status::NotSupported("Deserializing the specified CF option " +
name + " is not supported");
} else {
return Status::InvalidArgument(
"Unable to parse the specified CF option " + name);
}
}
} catch (const std::exception&) {
return Status::InvalidArgument("unable to parse the specified option " +
name);
}
return Status::OK();
}
#endif // ROCKSDB_LITE
ImmutableCFOptions::ImmutableCFOptions(const Options& options)
: ImmutableCFOptions(ImmutableDBOptions(options), options) {}

@ -9,6 +9,7 @@
#include "db/version_edit.h"
#include "logging/logging.h"
#include "options/options_helper.h"
#include "port/port.h"
#include "rocksdb/cache.h"
#include "rocksdb/env.h"
@ -17,6 +18,321 @@
#include "rocksdb/wal_filter.h"
namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE
std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::db_options_type_info = {
/*
// not yet supported
std::shared_ptr<Cache> row_cache;
std::shared_ptr<DeleteScheduler> delete_scheduler;
std::shared_ptr<Logger> info_log;
std::shared_ptr<RateLimiter> rate_limiter;
std::shared_ptr<Statistics> statistics;
std::vector<DbPath> db_paths;
std::vector<std::shared_ptr<EventListener>> listeners;
*/
{"advise_random_on_open",
{offsetof(struct DBOptions, advise_random_on_open),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"allow_mmap_reads",
{offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"allow_fallocate",
{offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"allow_mmap_writes",
{offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"use_direct_reads",
{offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"use_direct_writes",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"use_direct_io_for_flush_and_compaction",
{offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"allow_2pc",
{offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"allow_os_buffer",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"create_if_missing",
{offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"create_missing_column_families",
{offsetof(struct DBOptions, create_missing_column_families),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"disableDataSync",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"disable_data_sync", // for compatibility
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"enable_thread_tracking",
{offsetof(struct DBOptions, enable_thread_tracking),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"error_if_exists",
{offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"is_fd_close_on_exec",
{offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"paranoid_checks",
{offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"skip_log_error_on_recovery",
{offsetof(struct DBOptions, skip_log_error_on_recovery),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"skip_stats_update_on_db_open",
{offsetof(struct DBOptions, skip_stats_update_on_db_open),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"skip_checking_sst_file_sizes_on_db_open",
{offsetof(struct DBOptions, skip_checking_sst_file_sizes_on_db_open),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"new_table_reader_for_compaction_inputs",
{offsetof(struct DBOptions, new_table_reader_for_compaction_inputs),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"compaction_readahead_size",
{offsetof(struct DBOptions, compaction_readahead_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, compaction_readahead_size)}},
{"random_access_max_buffer_size",
{offsetof(struct DBOptions, random_access_max_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"use_adaptive_mutex",
{offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"use_fsync",
{offsetof(struct DBOptions, use_fsync), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"max_background_jobs",
{offsetof(struct DBOptions, max_background_jobs), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_background_jobs)}},
{"max_background_compactions",
{offsetof(struct DBOptions, max_background_compactions),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_background_compactions)}},
{"base_background_compactions",
{offsetof(struct DBOptions, base_background_compactions),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, base_background_compactions)}},
{"max_background_flushes",
{offsetof(struct DBOptions, max_background_flushes), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"max_file_opening_threads",
{offsetof(struct DBOptions, max_file_opening_threads),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"max_open_files",
{offsetof(struct DBOptions, max_open_files), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_open_files)}},
{"table_cache_numshardbits",
{offsetof(struct DBOptions, table_cache_numshardbits),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"db_write_buffer_size",
{offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"keep_log_file_num",
{offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"recycle_log_file_num",
{offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"log_file_time_to_roll",
{offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"manifest_preallocation_size",
{offsetof(struct DBOptions, manifest_preallocation_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"max_log_file_size",
{offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"db_log_dir",
{offsetof(struct DBOptions, db_log_dir), OptionType::kString,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"wal_dir",
{offsetof(struct DBOptions, wal_dir), OptionType::kString,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"max_subcompactions",
{offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"WAL_size_limit_MB",
{offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"WAL_ttl_seconds",
{offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"bytes_per_sync",
{offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, bytes_per_sync)}},
{"delayed_write_rate",
{offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, delayed_write_rate)}},
{"delete_obsolete_files_period_micros",
{offsetof(struct DBOptions, delete_obsolete_files_period_micros),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions,
delete_obsolete_files_period_micros)}},
{"max_manifest_file_size",
{offsetof(struct DBOptions, max_manifest_file_size),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"max_total_wal_size",
{offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_total_wal_size)}},
{"wal_bytes_per_sync",
{offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, wal_bytes_per_sync)}},
{"strict_bytes_per_sync",
{offsetof(struct DBOptions, strict_bytes_per_sync),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, strict_bytes_per_sync)}},
{"stats_dump_period_sec",
{offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, stats_dump_period_sec)}},
{"stats_persist_period_sec",
{offsetof(struct DBOptions, stats_persist_period_sec),
OptionType::kUInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, stats_persist_period_sec)}},
{"persist_stats_to_disk",
{offsetof(struct DBOptions, persist_stats_to_disk),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone,
offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}},
{"stats_history_buffer_size",
{offsetof(struct DBOptions, stats_history_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, stats_history_buffer_size)}},
{"fail_if_options_file_error",
{offsetof(struct DBOptions, fail_if_options_file_error),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"enable_pipelined_write",
{offsetof(struct DBOptions, enable_pipelined_write),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"unordered_write",
{offsetof(struct DBOptions, unordered_write), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"allow_concurrent_memtable_write",
{offsetof(struct DBOptions, allow_concurrent_memtable_write),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"wal_recovery_mode",
{offsetof(struct DBOptions, wal_recovery_mode),
OptionType::kWALRecoveryMode, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"enable_write_thread_adaptive_yield",
{offsetof(struct DBOptions, enable_write_thread_adaptive_yield),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"write_thread_slow_yield_usec",
{offsetof(struct DBOptions, write_thread_slow_yield_usec),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"max_write_batch_group_size_bytes",
{offsetof(struct DBOptions, max_write_batch_group_size_bytes),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"write_thread_max_yield_usec",
{offsetof(struct DBOptions, write_thread_max_yield_usec),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"access_hint_on_compaction_start",
{offsetof(struct DBOptions, access_hint_on_compaction_start),
OptionType::kAccessHint, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"info_log_level",
{offsetof(struct DBOptions, info_log_level), OptionType::kInfoLogLevel,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"dump_malloc_stats",
{offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"avoid_flush_during_recovery",
{offsetof(struct DBOptions, avoid_flush_during_recovery),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"avoid_flush_during_shutdown",
{offsetof(struct DBOptions, avoid_flush_during_shutdown),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}},
{"writable_file_max_buffer_size",
{offsetof(struct DBOptions, writable_file_max_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}},
{"allow_ingest_behind",
{offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone,
offsetof(struct ImmutableDBOptions, allow_ingest_behind)}},
{"preserve_deletes",
{offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone,
offsetof(struct ImmutableDBOptions, preserve_deletes)}},
{"concurrent_prepare", // Deprecated by two_write_queues
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"two_write_queues",
{offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone,
offsetof(struct ImmutableDBOptions, two_write_queues)}},
{"manual_wal_flush",
{offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone,
offsetof(struct ImmutableDBOptions, manual_wal_flush)}},
{"seq_per_batch",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"atomic_flush",
{offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone,
offsetof(struct ImmutableDBOptions, atomic_flush)}},
{"avoid_unnecessary_blocking_io",
{offsetof(struct DBOptions, avoid_unnecessary_blocking_io),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone,
offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}},
{"write_dbid_to_manifest",
{offsetof(struct DBOptions, write_dbid_to_manifest),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"log_readahead_size",
{offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"best_efforts_recovery",
{offsetof(struct DBOptions, best_efforts_recovery),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
};
#endif // ROCKSDB_LITE
ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {}

File diff suppressed because it is too large Load Diff

@ -12,6 +12,7 @@
#include "options/cf_options.h"
#include "options/db_options.h"
#include "options/options_type.h"
#include "rocksdb/options.h"
#include "rocksdb/status.h"
#include "rocksdb/table.h"
@ -28,6 +29,11 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
#ifndef ROCKSDB_LITE
Status ParseColumnFamilyOption(const std::string& name,
const std::string& org_value,
ColumnFamilyOptions* new_options,
bool input_strings_escaped = false);
Status GetMutableOptionsFromStrings(
const MutableCFOptions& base_options,
const std::unordered_map<std::string, std::string>& options_map,
@ -44,71 +50,9 @@ Status GetTableFactoryFromMap(
std::shared_ptr<TableFactory>* table_factory,
bool ignore_unknown_options = false);
enum class OptionType {
kBoolean,
kInt,
kInt32T,
kInt64T,
kVectorInt,
kUInt,
kUInt32T,
kUInt64T,
kSizeT,
kString,
kDouble,
kCompactionStyle,
kCompactionPri,
kSliceTransform,
kCompressionType,
kVectorCompressionType,
kTableFactory,
kComparator,
kCompactionFilter,
kCompactionFilterFactory,
kCompactionOptionsFIFO,
kCompactionOptionsUniversal,
kCompactionStopStyle,
kMergeOperator,
kMemTableRepFactory,
kBlockBasedTableIndexType,
kBlockBasedTableDataBlockIndexType,
kBlockBasedTableIndexShorteningMode,
kFilterPolicy,
kFlushBlockPolicyFactory,
kChecksumType,
kEncodingType,
kWALRecoveryMode,
kAccessHint,
kInfoLogLevel,
kLRUCacheOptions,
kEnv,
kUnknown,
};
enum class OptionVerificationType {
kNormal,
kByName, // The option is pointer typed so we can only verify
// based on it's name.
kByNameAllowNull, // Same as kByName, but it also allows the case
// where one of them is a nullptr.
kByNameAllowFromNull, // Same as kByName, but it also allows the case
// where the old option is nullptr.
kDeprecated // The option is no longer used in rocksdb. The RocksDB
// OptionsParser will still accept this option if it
// happen to exists in some Options file. However,
// the parser will not include it in serialization
// and verification processes.
};
// A struct for storing constant option information such as option name,
// option type, and offset.
struct OptionTypeInfo {
int offset;
OptionType type;
OptionVerificationType verification;
bool is_mutable;
int mutable_offset;
};
Status ParseCompressionOptions(const std::string& value,
const std::string& name,
CompressionOptions& compression_opts);
// A helper function that converts "opt_address" to a std::string
// based on the specified OptionType.
@ -145,6 +89,10 @@ extern Status StringToMap(
extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
const std::string& value);
Status GetStringFromStruct(
std::string* opt_string, const void* const options,
const std::unordered_map<std::string, OptionTypeInfo>& type_info,
const std::string& delimiter);
#endif // !ROCKSDB_LITE
struct OptionsHelper {

@ -620,10 +620,7 @@ bool AreEqualOptions(
return false;
}
default:
if (type_info.verification == OptionVerificationType::kByName ||
type_info.verification ==
OptionVerificationType::kByNameAllowFromNull ||
type_info.verification == OptionVerificationType::kByNameAllowNull) {
if (type_info.IsByName()) {
std::string value1;
bool result =
SerializeSingleOptionHelper(offset1, type_info.type, &value1);
@ -637,13 +634,12 @@ bool AreEqualOptions(
if (iter == opt_map->end()) {
return true;
} else {
if (type_info.verification ==
OptionVerificationType::kByNameAllowNull) {
if (type_info.IsEnabled(OptionVerificationType::kByNameAllowNull)) {
if (iter->second == kNullptrString || value1 == kNullptrString) {
return true;
}
} else if (type_info.verification ==
OptionVerificationType::kByNameAllowFromNull) {
} else if (type_info.IsEnabled(
OptionVerificationType::kByNameAllowFromNull)) {
if (iter->second == kNullptrString) {
return true;
}
@ -684,7 +680,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
// Verify ColumnFamily Name
if (cf_names.size() != parser.cf_names()->size()) {
if (sanity_check_level >= kSanityLevelLooselyCompatible) {
if (sanity_check_level >=
OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) {
return Status::InvalidArgument(
"[RocksDBOptionParser Error] The persisted options does not have "
"the same number of column family names as the db instance.");
@ -706,7 +703,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
// Verify Column Family Options
if (cf_opts.size() != parser.cf_opts()->size()) {
if (sanity_check_level >= kSanityLevelLooselyCompatible) {
if (sanity_check_level >=
OptionsSanityCheckLevel::kSanityLevelLooselyCompatible) {
return Status::InvalidArgument(
"[RocksDBOptionsParser Error]",
"The persisted options does not have the same number of "
@ -740,7 +738,7 @@ Status RocksDBOptionsParser::VerifyDBOptions(
const std::unordered_map<std::string, std::string>* /*opt_map*/,
OptionsSanityCheckLevel sanity_check_level) {
for (const auto& pair : db_options_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
if (pair.second.IsDeprecated()) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
@ -778,7 +776,7 @@ Status RocksDBOptionsParser::VerifyCFOptions(
const std::unordered_map<std::string, std::string>* persisted_opt_map,
OptionsSanityCheckLevel sanity_check_level) {
for (const auto& pair : cf_options_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
if (pair.second.IsDeprecated()) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;
@ -814,7 +812,7 @@ Status RocksDBOptionsParser::VerifyTableFactory(
const TableFactory* base_tf, const TableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level) {
if (base_tf && file_tf) {
if (sanity_check_level > kSanityLevelNone &&
if (sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone &&
std::string(base_tf->Name()) != std::string(file_tf->Name())) {
return Status::Corruption(
"[RocksDBOptionsParser]: "

@ -0,0 +1,141 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE {
enum class OptionType {
kBoolean,
kInt,
kInt32T,
kInt64T,
kVectorInt,
kUInt,
kUInt32T,
kUInt64T,
kSizeT,
kString,
kDouble,
kCompactionStyle,
kCompactionPri,
kSliceTransform,
kCompressionType,
kVectorCompressionType,
kTableFactory,
kComparator,
kCompactionFilter,
kCompactionFilterFactory,
kCompactionOptionsFIFO,
kCompactionOptionsUniversal,
kCompactionStopStyle,
kMergeOperator,
kMemTableRepFactory,
kBlockBasedTableIndexType,
kBlockBasedTableDataBlockIndexType,
kBlockBasedTableIndexShorteningMode,
kFilterPolicy,
kFlushBlockPolicyFactory,
kChecksumType,
kEncodingType,
kWALRecoveryMode,
kAccessHint,
kInfoLogLevel,
kLRUCacheOptions,
kEnv,
kUnknown,
};
enum class OptionVerificationType {
kNormal,
kByName, // The option is pointer typed so we can only verify
// based on it's name.
kByNameAllowNull, // Same as kByName, but it also allows the case
// where one of them is a nullptr.
kByNameAllowFromNull, // Same as kByName, but it also allows the case
// where the old option is nullptr.
kDeprecated // The option is no longer used in rocksdb. The RocksDB
// OptionsParser will still accept this option if it
// happen to exists in some Options file. However,
// the parser will not include it in serialization
// and verification processes.
};
enum class OptionTypeFlags : uint32_t {
kNone = 0x00, // No flags
kMutable = 0x01, // Option is mutable
};
inline OptionTypeFlags operator|(const OptionTypeFlags &a,
const OptionTypeFlags &b) {
return static_cast<OptionTypeFlags>(static_cast<uint32_t>(a) |
static_cast<uint32_t>(b));
}
inline OptionTypeFlags operator&(const OptionTypeFlags &a,
const OptionTypeFlags &b) {
return static_cast<OptionTypeFlags>(static_cast<uint32_t>(a) &
static_cast<uint32_t>(b));
}
// A struct for storing constant option information such as option name,
// option type, and offset.
class OptionTypeInfo {
public:
int offset;
int mutable_offset;
OptionType type;
// A simple "normal", non-mutable Type "_type" at _offset
OptionTypeInfo(int _offset, OptionType _type)
: offset(_offset),
mutable_offset(0),
type(_type),
verification(OptionVerificationType::kNormal),
flags(OptionTypeFlags::kNone) {}
// A simple "normal", mutable Type "_type" at _offset
OptionTypeInfo(int _offset, OptionType _type, int _mutable_offset)
: offset(_offset),
mutable_offset(_mutable_offset),
type(_type),
verification(OptionVerificationType::kNormal),
flags(OptionTypeFlags::kMutable) {}
OptionTypeInfo(int _offset, OptionType _type,
OptionVerificationType _verification, OptionTypeFlags _flags,
int _mutable_offset)
: offset(_offset),
mutable_offset(_mutable_offset),
type(_type),
verification(_verification),
flags(_flags) {}
bool IsEnabled(OptionTypeFlags otf) const { return (flags & otf) == otf; }
bool IsMutable() const { return IsEnabled(OptionTypeFlags::kMutable); }
bool IsDeprecated() const {
return IsEnabled(OptionVerificationType::kDeprecated);
}
bool IsEnabled(OptionVerificationType ovf) const {
return verification == ovf;
}
bool IsByName() const {
return (verification == OptionVerificationType::kByName ||
verification == OptionVerificationType::kByNameAllowNull ||
verification == OptionVerificationType::kByNameAllowFromNull);
}
protected:
private:
OptionVerificationType verification;
OptionTypeFlags flags;
};
} // namespace ROCKSDB_NAMESPACE

@ -157,6 +157,126 @@ size_t TailPrefetchStats::GetSuggestedPrefetchSize() {
return std::min(kMaxPrefetchSize, max_qualified_size);
}
#ifndef ROCKSDB_LITE
static std::unordered_map<std::string, OptionTypeInfo>
block_based_table_type_info = {
/* currently not supported
std::shared_ptr<Cache> block_cache = nullptr;
std::shared_ptr<Cache> block_cache_compressed = nullptr;
*/
{"flush_block_policy_factory",
{offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
{"cache_index_and_filter_blocks",
{offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"cache_index_and_filter_blocks_with_high_priority",
{offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks_with_high_priority),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"pin_l0_filter_and_index_blocks_in_cache",
{offsetof(struct BlockBasedTableOptions,
pin_l0_filter_and_index_blocks_in_cache),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"index_type",
{offsetof(struct BlockBasedTableOptions, index_type),
OptionType::kBlockBasedTableIndexType,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"hash_index_allow_collision",
{offsetof(struct BlockBasedTableOptions, hash_index_allow_collision),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"data_block_index_type",
{offsetof(struct BlockBasedTableOptions, data_block_index_type),
OptionType::kBlockBasedTableDataBlockIndexType,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"index_shortening",
{offsetof(struct BlockBasedTableOptions, index_shortening),
OptionType::kBlockBasedTableIndexShorteningMode,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"data_block_hash_table_util_ratio",
{offsetof(struct BlockBasedTableOptions,
data_block_hash_table_util_ratio),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"checksum",
{offsetof(struct BlockBasedTableOptions, checksum),
OptionType::kChecksumType, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"no_block_cache",
{offsetof(struct BlockBasedTableOptions, no_block_cache),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"block_size",
{offsetof(struct BlockBasedTableOptions, block_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"block_size_deviation",
{offsetof(struct BlockBasedTableOptions, block_size_deviation),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"block_restart_interval",
{offsetof(struct BlockBasedTableOptions, block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"index_block_restart_interval",
{offsetof(struct BlockBasedTableOptions, index_block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"index_per_partition",
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"metadata_block_size",
{offsetof(struct BlockBasedTableOptions, metadata_block_size),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"partition_filters",
{offsetof(struct BlockBasedTableOptions, partition_filters),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"filter_policy",
{offsetof(struct BlockBasedTableOptions, filter_policy),
OptionType::kFilterPolicy, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
{"whole_key_filtering",
{offsetof(struct BlockBasedTableOptions, whole_key_filtering),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"skip_table_builder_flush",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}},
{"format_version",
{offsetof(struct BlockBasedTableOptions, format_version),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"verify_compression",
{offsetof(struct BlockBasedTableOptions, verify_compression),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"read_amp_bytes_per_bit",
{offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"enable_index_compression",
{offsetof(struct BlockBasedTableOptions, enable_index_compression),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"block_align",
{offsetof(struct BlockBasedTableOptions, block_align),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"pin_top_level_index_and_filter",
{offsetof(struct BlockBasedTableOptions,
pin_top_level_index_and_filter),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}};
#endif // ROCKSDB_LITE
// TODO(myabandeh): We should return an error instead of silently changing the
// options
BlockBasedTableFactory::BlockBasedTableFactory(
@ -416,46 +536,13 @@ std::string BlockBasedTableFactory::GetPrintableTableOptions() const {
}
#ifndef ROCKSDB_LITE
namespace {
bool SerializeSingleBlockBasedTableOption(
std::string* opt_string, const BlockBasedTableOptions& bbt_options,
const std::string& name, const std::string& delimiter) {
auto iter = block_based_table_type_info.find(name);
if (iter == block_based_table_type_info.end()) {
return false;
}
auto& opt_info = iter->second;
const char* opt_address =
reinterpret_cast<const char*>(&bbt_options) + opt_info.offset;
std::string value;
bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
if (result) {
*opt_string = name + "=" + value + delimiter;
}
return result;
}
} // namespace
Status BlockBasedTableFactory::GetOptionString(
std::string* opt_string, const std::string& delimiter) const {
assert(opt_string);
opt_string->clear();
for (auto iter = block_based_table_type_info.begin();
iter != block_based_table_type_info.end(); ++iter) {
if (iter->second.verification == OptionVerificationType::kDeprecated) {
// If the option is no longer used in rocksdb and marked as deprecated,
// we skip it in the serialization.
continue;
}
std::string single_output;
bool result = SerializeSingleBlockBasedTableOption(
&single_output, table_options_, iter->first, delimiter);
assert(result);
if (result) {
opt_string->append(single_output);
}
}
return Status::OK();
return GetStringFromStruct(opt_string, &table_options_,
block_based_table_type_info, delimiter);
}
#else
Status BlockBasedTableFactory::GetOptionString(
@ -535,7 +622,7 @@ std::string ParseBlockBasedTableOption(const std::string& name,
}
}
const auto& opt_info = iter->second;
if (opt_info.verification != OptionVerificationType::kDeprecated &&
if (!opt_info.IsDeprecated() &&
!ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
opt_info.type, value)) {
return "Invalid value";
@ -574,12 +661,7 @@ Status GetBlockBasedTableOptionsFromMap(
!input_strings_escaped || // !input_strings_escaped indicates
// the old API, where everything is
// parsable.
(iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification !=
OptionVerificationType::kByNameAllowNull &&
iter->second.verification !=
OptionVerificationType::kByNameAllowFromNull &&
iter->second.verification != OptionVerificationType::kDeprecated)) {
(!iter->second.IsByName() && !iter->second.IsDeprecated())) {
// Restore "new_options" to the default "base_options".
*new_table_options = table_options;
return Status::InvalidArgument("Can't parse BlockBasedTableOptions:",
@ -595,7 +677,7 @@ Status VerifyBlockBasedTableFactory(
const BlockBasedTableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level) {
if ((base_tf != nullptr) != (file_tf != nullptr) &&
sanity_check_level > kSanityLevelNone) {
sanity_check_level > OptionsSanityCheckLevel::kSanityLevelNone) {
return Status::Corruption(
"[RocksDBOptionsParser]: Inconsistent TableFactory class type");
}
@ -608,7 +690,7 @@ Status VerifyBlockBasedTableFactory(
const auto& file_opt = file_tf->table_options();
for (auto& pair : block_based_table_type_info) {
if (pair.second.verification == OptionVerificationType::kDeprecated) {
if (pair.second.IsDeprecated()) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
continue;

@ -93,103 +93,5 @@ extern Status VerifyBlockBasedTableFactory(
const BlockBasedTableFactory* file_tf,
OptionsSanityCheckLevel sanity_check_level);
static std::unordered_map<std::string, OptionTypeInfo>
block_based_table_type_info = {
/* currently not supported
std::shared_ptr<Cache> block_cache = nullptr;
std::shared_ptr<Cache> block_cache_compressed = nullptr;
*/
{"flush_block_policy_factory",
{offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName,
false, 0}},
{"cache_index_and_filter_blocks",
{offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"cache_index_and_filter_blocks_with_high_priority",
{offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks_with_high_priority),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"pin_l0_filter_and_index_blocks_in_cache",
{offsetof(struct BlockBasedTableOptions,
pin_l0_filter_and_index_blocks_in_cache),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"index_type",
{offsetof(struct BlockBasedTableOptions, index_type),
OptionType::kBlockBasedTableIndexType,
OptionVerificationType::kNormal, false, 0}},
{"hash_index_allow_collision",
{offsetof(struct BlockBasedTableOptions, hash_index_allow_collision),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"data_block_index_type",
{offsetof(struct BlockBasedTableOptions, data_block_index_type),
OptionType::kBlockBasedTableDataBlockIndexType,
OptionVerificationType::kNormal, false, 0}},
{"index_shortening",
{offsetof(struct BlockBasedTableOptions, index_shortening),
OptionType::kBlockBasedTableIndexShorteningMode,
OptionVerificationType::kNormal, false, 0}},
{"data_block_hash_table_util_ratio",
{offsetof(struct BlockBasedTableOptions,
data_block_hash_table_util_ratio),
OptionType::kDouble, OptionVerificationType::kNormal, false, 0}},
{"checksum",
{offsetof(struct BlockBasedTableOptions, checksum),
OptionType::kChecksumType, OptionVerificationType::kNormal, false,
0}},
{"no_block_cache",
{offsetof(struct BlockBasedTableOptions, no_block_cache),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"block_size",
{offsetof(struct BlockBasedTableOptions, block_size),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
{"block_size_deviation",
{offsetof(struct BlockBasedTableOptions, block_size_deviation),
OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
{"block_restart_interval",
{offsetof(struct BlockBasedTableOptions, block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
{"index_block_restart_interval",
{offsetof(struct BlockBasedTableOptions, index_block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal, false, 0}},
{"index_per_partition",
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, false,
0}},
{"metadata_block_size",
{offsetof(struct BlockBasedTableOptions, metadata_block_size),
OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
{"partition_filters",
{offsetof(struct BlockBasedTableOptions, partition_filters),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"filter_policy",
{offsetof(struct BlockBasedTableOptions, filter_policy),
OptionType::kFilterPolicy, OptionVerificationType::kByName, false,
0}},
{"whole_key_filtering",
{offsetof(struct BlockBasedTableOptions, whole_key_filtering),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"skip_table_builder_flush",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, false,
0}},
{"format_version",
{offsetof(struct BlockBasedTableOptions, format_version),
OptionType::kUInt32T, OptionVerificationType::kNormal, false, 0}},
{"verify_compression",
{offsetof(struct BlockBasedTableOptions, verify_compression),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"read_amp_bytes_per_bit",
{offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
{"enable_index_compression",
{offsetof(struct BlockBasedTableOptions, enable_index_compression),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"block_align",
{offsetof(struct BlockBasedTableOptions, block_align),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"pin_top_level_index_and_filter",
{offsetof(struct BlockBasedTableOptions,
pin_top_level_index_and_filter),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}};
#endif // !ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE

@ -17,6 +17,34 @@
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
static std::unordered_map<std::string, OptionTypeInfo> plain_table_type_info = {
{"user_key_len",
{offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"bloom_bits_per_key",
{offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"hash_table_ratio",
{offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"index_sparseness",
{offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"huge_page_tlb_size",
{offsetof(struct PlainTableOptions, huge_page_tlb_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}},
{"encoding_type",
{offsetof(struct PlainTableOptions, encoding_type),
OptionType::kEncodingType, OptionVerificationType::kByName,
OptionTypeFlags::kNone, 0}},
{"full_scan_mode",
{offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"store_index_in_file",
{offsetof(struct PlainTableOptions, store_index_in_file),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}};
Status PlainTableFactory::NewTableReader(
const TableReaderOptions& table_reader_options,
@ -178,7 +206,7 @@ std::string ParsePlainTableOptions(const std::string& name,
}
}
const auto& opt_info = iter->second;
if (opt_info.verification != OptionVerificationType::kDeprecated &&
if (!opt_info.IsDeprecated() &&
!ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
opt_info.type, value)) {
return "Invalid value";
@ -202,12 +230,7 @@ Status GetPlainTableOptionsFromMap(
!input_strings_escaped || // !input_strings_escaped indicates
// the old API, where everything is
// parsable.
(iter->second.verification != OptionVerificationType::kByName &&
iter->second.verification !=
OptionVerificationType::kByNameAllowNull &&
iter->second.verification !=
OptionVerificationType::kByNameAllowFromNull &&
iter->second.verification != OptionVerificationType::kDeprecated)) {
(!iter->second.IsByName() && !iter->second.IsDeprecated())) {
// Restore "new_options" to the default "base_options".
*new_table_options = table_options;
return Status::InvalidArgument("Can't parse PlainTableOptions:",

@ -193,31 +193,6 @@ class PlainTableFactory : public TableFactory {
PlainTableOptions table_options_;
};
static std::unordered_map<std::string, OptionTypeInfo> plain_table_type_info = {
{"user_key_len",
{offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T,
OptionVerificationType::kNormal, false, 0}},
{"bloom_bits_per_key",
{offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt,
OptionVerificationType::kNormal, false, 0}},
{"hash_table_ratio",
{offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble,
OptionVerificationType::kNormal, false, 0}},
{"index_sparseness",
{offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT,
OptionVerificationType::kNormal, false, 0}},
{"huge_page_tlb_size",
{offsetof(struct PlainTableOptions, huge_page_tlb_size),
OptionType::kSizeT, OptionVerificationType::kNormal, false, 0}},
{"encoding_type",
{offsetof(struct PlainTableOptions, encoding_type),
OptionType::kEncodingType, OptionVerificationType::kByName, false, 0}},
{"full_scan_mode",
{offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean,
OptionVerificationType::kNormal, false, 0}},
{"store_index_in_file",
{offsetof(struct PlainTableOptions, store_index_in_file),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}};
} // namespace ROCKSDB_NAMESPACE
#endif // ROCKSDB_LITE

Loading…
Cancel
Save