[RocksDB Options File] Add TableOptions section and support BlockBasedTable

Summary:
Introduce TableOptions section and support BlockBasedTable in RocksDB
options file.  A TableOptions section has the following format:

  [TableOptions/<FactoryClassName> "<ColumnFamily Name>"]

which includes information about its TableFactory class and belonging
column family.  Below is an example TableOptions section of a
BlockBasedTableOptions that belongs to the default column family:

  [TableOptions/BlockBasedTable "default"]
    format_version=0
    whole_key_filtering=true
    block_size_deviation=10
    block_size=4096
    block_restart_interval=16
    filter_policy=nullptr
    no_block_cache=false
    checksum=kCRC32c
    cache_index_and_filter_blocks=false
    index_type=kBinarySearch
    hash_index_allow_collision=true
    flush_block_policy_factory=FlushBlockBySizePolicyFactory

Currently, Cache-type options (i.e., block_cache and block_cache_compressed)
are not supported.

Test Plan: options_test

Reviewers: igor, anthony, IslamAbdelRahman, sdong

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D48435

examples/rocksdb_option_file_example.ini

@@ -29,25 +29,167 @@
 #
 # Below is an example of a RocksDB options file:
 [Version]
-  # The Version section stores the version information about rocksdb
+  rocksdb_version=4.0.0
-  # and option file.  This is used for handling potential format
+  options_file_version=1.1
-  # change in the future.
+
-  rocksdb_version=4.0.0  # We support "#" style comment.
-  options_file_version=1.0
 [DBOptions]
-  # Followed by the Version section is the DBOptions section.
+  stats_dump_period_sec=600
-  # The value of an options can be assigned using a statement.
+  max_manifest_file_size=18446744073709551615
-  # Note that for those options that is not set in the options file,
+  bytes_per_sync=0
-  # we will use the default value.
+  delayed_write_rate=1048576
-  max_open_files=12345
+  WAL_ttl_seconds=0
-  max_background_flushes=301
+  WAL_size_limit_MB=0
+  max_subcompactions=1
+  wal_dir=
+  wal_bytes_per_sync=0
+  db_write_buffer_size=0
+  max_total_wal_size=0
+  skip_stats_update_on_db_open=false
+  max_open_files=5000
+  max_file_opening_threads=1
+  use_fsync=false
+  max_background_compactions=1
+  manifest_preallocation_size=4194304
+  max_background_flushes=1
+  is_fd_close_on_exec=true
+  create_if_missing=false
+  use_adaptive_mutex=false
+  enable_thread_tracking=false
+  disableDataSync=false
+  max_log_file_size=0
+  advise_random_on_open=true
+  create_missing_column_families=false
+  keep_log_file_num=1000
+  table_cache_numshardbits=4
+  error_if_exists=false
+  skip_log_error_on_recovery=false
+  allow_os_buffer=true
+  allow_mmap_reads=false
+  paranoid_checks=true
+  delete_obsolete_files_period_micros=21600000000
+  disable_data_sync=false
+  log_file_time_to_roll=0
+  compaction_readahead_size=0
+  db_log_dir=
+  new_table_reader_for_compaction_inputs=false
+  allow_mmap_writes=false
+
 [CFOptions "default"]
-  # ColumnFamilyOptions section must follow the format of
+  compaction_style=kCompactionStyleLevel
-  # [CFOptions "cf name"].  If a rocksdb instance
+  compaction_filter=nullptr
-  # has multiple column families, then its CFOptions must be
+  num_levels=7
-  # specified in the same order as column family creation order.
+  table_factory=BlockBasedTable
-[CFOptions "the second column family"]
+  comparator=leveldb.BytewiseComparator
-  # Each column family must have one section in the RocksDB option
+  max_sequential_skip_in_iterations=8
-  # file even all the options of this column family are set to
+  soft_rate_limit=0.000000
-  # default value.
+  max_bytes_for_level_base=536870912
-[CFOptions "the third column family"]
+  memtable_prefix_bloom_probes=6
+  memtable_prefix_bloom_bits=0
+  memtable_prefix_bloom_huge_page_tlb_size=0
+  max_successive_merges=0
+  arena_block_size=0
+  min_write_buffer_number_to_merge=2
+  target_file_size_multiplier=1
+  source_compaction_factor=1
+  max_bytes_for_level_multiplier=10
+  compaction_filter_factory=nullptr
+  max_write_buffer_number=6
+  level0_stop_writes_trigger=24
+  compression=kSnappyCompression
+  level0_file_num_compaction_trigger=2
+  purge_redundant_kvs_while_flush=true
+  max_write_buffer_number_to_maintain=0
+  memtable_factory=SkipListFactory
+  max_grandparent_overlap_factor=10
+  expanded_compaction_factor=25
+  hard_pending_compaction_bytes_limit=0
+  inplace_update_num_locks=10000
+  level_compaction_dynamic_level_bytes=false
+  level0_slowdown_writes_trigger=20
+  filter_deletes=false
+  verify_checksums_in_compaction=true
+  min_partial_merge_operands=2
+  paranoid_file_checks=false
+  target_file_size_base=67108864
+  optimize_filters_for_hits=false
+  merge_operator=nullptr
+  compression_per_level=kNoCompression:kNoCompression:kSnappyCompression:kSnappyCompression:kSnappyCompression:kSnappyCompression:kSnappyCompression
+  compaction_measure_io_stats=false
+  prefix_extractor=nullptr
+  bloom_locality=0
+  write_buffer_size=134217728
+  disable_auto_compactions=false
+  inplace_update_support=false
+[TableOptions/BlockBasedTable "default"]
+  format_version=0
+  whole_key_filtering=true
+  block_size_deviation=10
+  block_size=4096
+  block_restart_interval=16
+  filter_policy=nullptr
+  no_block_cache=false
+  checksum=kCRC32c
+  cache_index_and_filter_blocks=false
+  index_type=kBinarySearch
+  hash_index_allow_collision=true
+  flush_block_policy_factory=FlushBlockBySizePolicyFactory
+
+[CFOptions "universal"]
+  compaction_style=kCompactionStyleUniversal
+  compaction_filter=nullptr
+  num_levels=7
+  table_factory=BlockBasedTable
+  comparator=leveldb.BytewiseComparator
+  max_sequential_skip_in_iterations=8
+  soft_rate_limit=0.000000
+  max_bytes_for_level_base=10485760
+  memtable_prefix_bloom_probes=6
+  memtable_prefix_bloom_bits=0
+  memtable_prefix_bloom_huge_page_tlb_size=0
+  max_successive_merges=0
+  arena_block_size=0
+  min_write_buffer_number_to_merge=2
+  target_file_size_multiplier=1
+  source_compaction_factor=1
+  max_bytes_for_level_multiplier=10
+  compaction_filter_factory=nullptr
+  max_write_buffer_number=6
+  level0_stop_writes_trigger=24
+  compression=kSnappyCompression
+  level0_file_num_compaction_trigger=4
+  purge_redundant_kvs_while_flush=true
+  max_write_buffer_number_to_maintain=0
+  memtable_factory=SkipListFactory
+  max_grandparent_overlap_factor=10
+  expanded_compaction_factor=25
+  hard_pending_compaction_bytes_limit=0
+  inplace_update_num_locks=10000
+  level_compaction_dynamic_level_bytes=false
+  level0_slowdown_writes_trigger=20
+  filter_deletes=false
+  verify_checksums_in_compaction=true
+  min_partial_merge_operands=2
+  paranoid_file_checks=false
+  target_file_size_base=2097152
+  optimize_filters_for_hits=false
+  merge_operator=nullptr
+  compression_per_level=
+  compaction_measure_io_stats=false
+  prefix_extractor=nullptr
+  bloom_locality=0
+  write_buffer_size=134217728
+  disable_auto_compactions=false
+  inplace_update_support=false
+[TableOptions/BlockBasedTable "universal"]
+  format_version=0
+  whole_key_filtering=true
+  block_size_deviation=10
+  block_size=4096
+  block_restart_interval=16
+  filter_policy=nullptr
+  no_block_cache=false
+  checksum=kCRC32c
+  cache_index_and_filter_blocks=false
+  index_type=kBinarySearch
+  hash_index_allow_collision=true
+  flush_block_policy_factory=FlushBlockBySizePolicyFactory

include/rocksdb/convenience.h

@@ -40,7 +40,8 @@ Status GetDBOptionsFromMap(
 Status GetBlockBasedTableOptionsFromMap(
     const BlockBasedTableOptions& table_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    BlockBasedTableOptions* new_table_options);
+    BlockBasedTableOptions* new_table_options,
+    bool input_strings_escaped = false);
 
 // Take a string representation of option names and  values, apply them into the
 // base_options, and return the new options as a result. The string has the

util/options_helper.cc

@@ -174,26 +174,52 @@ bool ParseCompressionType(const std::string& string_value,
   return true;
 }
 
-BlockBasedTableOptions::IndexType ParseBlockBasedTableIndexType(
+bool SerializeBlockBasedTableIndexType(
-    const std::string& type) {
+    const BlockBasedTableOptions::IndexType& type, std::string* value) {
+  switch (type) {
+    case BlockBasedTableOptions::kBinarySearch:
+      *value = "kBinarySearch";
+      return true;
+    case BlockBasedTableOptions::kHashSearch:
+      *value = "kHashSearch";
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool ParseBlockBasedTableIndexType(const std::string& type,
+                                   BlockBasedTableOptions::IndexType* value) {
   if (type == "kBinarySearch") {
-    return BlockBasedTableOptions::kBinarySearch;
+    *value = BlockBasedTableOptions::kBinarySearch;
   } else if (type == "kHashSearch") {
-    return BlockBasedTableOptions::kHashSearch;
+    *value = BlockBasedTableOptions::kHashSearch;
+  } else {
+    return false;
+  }
+  return true;
+}
+
+static std::unordered_map<std::string, ChecksumType> checksum_type_map = {
+    {"kNoChecksum", kNoChecksum}, {"kCRC32c", kCRC32c}, {"kxxHash", kxxHash}};
+
+bool ParseChecksumType(const std::string& type, ChecksumType* value) {
+  auto iter = checksum_type_map.find(type);
+  if (iter != checksum_type_map.end()) {
+    *value = iter->second;
+    return true;
   }
-  throw std::invalid_argument("Unknown index type: " + type);
+  return false;
 }
 
-ChecksumType ParseBlockBasedTableChecksumType(
+bool SerializeChecksumType(const ChecksumType& type, std::string* value) {
-    const std::string& type) {
+  for (const auto& pair : checksum_type_map) {
-  if (type == "kNoChecksum") {
+    if (pair.second == type) {
-    return kNoChecksum;
+      *value = pair.first;
-  } else if (type == "kCRC32c") {
+      return true;
-    return kCRC32c;
+    }
-  } else if (type == "kxxHash") {
-    return kxxHash;
   }
-  throw std::invalid_argument("Unknown checksum type: " + type);
+  return false;
 }
 
 bool ParseBoolean(const std::string& type, const std::string& value) {
@@ -328,6 +354,7 @@ bool ParseSliceTransformHelper(
     const std::string& kFixedPrefixName, const std::string& kCappedPrefixName,
     const std::string& value,
     std::shared_ptr<const SliceTransform>* slice_transform) {
+  static const std::string kNullptrString = "nullptr";
   auto& pe_value = value;
   if (pe_value.size() > kFixedPrefixName.size() &&
       pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
@@ -339,7 +366,7 @@ bool ParseSliceTransformHelper(
     int prefix_length =
         ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
     slice_transform->reset(NewCappedPrefixTransform(prefix_length));
-  } else if (value == "nullptr") {
+  } else if (value == kNullptrString) {
     slice_transform->reset();
   } else {
     return false;
@@ -414,6 +441,13 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
       return ParseSliceTransform(
           value, reinterpret_cast<std::shared_ptr<const SliceTransform>*>(
                      opt_address));
+    case OptionType::kChecksumType:
+      return ParseChecksumType(value,
+                               reinterpret_cast<ChecksumType*>(opt_address));
+    case OptionType::kBlockBasedTableIndexType:
+      return ParseBlockBasedTableIndexType(
+          value,
+          reinterpret_cast<BlockBasedTableOptions::IndexType*>(opt_address));
     default:
       return false;
   }
@@ -425,6 +459,7 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
 bool SerializeSingleOptionHelper(const char* opt_address,
                                  const OptionType opt_type,
                                  std::string* value) {
+  static const std::string kNullptrString = kNullptrString;
   assert(value);
   switch (opt_type) {
     case OptionType::kBoolean:
@@ -469,7 +504,7 @@ bool SerializeSingleOptionHelper(const char* opt_address,
           reinterpret_cast<const std::shared_ptr<const SliceTransform>*>(
               opt_address);
       *value = slice_transform_ptr->get() ? slice_transform_ptr->get()->Name()
-                                          : "nullptr";
+                                          : kNullptrString;
       break;
     }
     case OptionType::kTableFactory: {
@@ -477,40 +512,61 @@ bool SerializeSingleOptionHelper(const char* opt_address,
           reinterpret_cast<const std::shared_ptr<const TableFactory>*>(
               opt_address);
       *value = table_factory_ptr->get() ? table_factory_ptr->get()->Name()
-                                        : "nullptr";
+                                        : kNullptrString;
       break;
     }
     case OptionType::kComparator: {
       // it's a const pointer of const Comparator*
       const auto* ptr = reinterpret_cast<const Comparator* const*>(opt_address);
-      *value = *ptr ? (*ptr)->Name() : "nullptr";
+      *value = *ptr ? (*ptr)->Name() : kNullptrString;
       break;
     }
     case OptionType::kCompactionFilter: {
       // it's a const pointer of const CompactionFilter*
       const auto* ptr =
           reinterpret_cast<const CompactionFilter* const*>(opt_address);
-      *value = *ptr ? (*ptr)->Name() : "nullptr";
+      *value = *ptr ? (*ptr)->Name() : kNullptrString;
       break;
     }
     case OptionType::kCompactionFilterFactory: {
       const auto* ptr =
           reinterpret_cast<const std::shared_ptr<CompactionFilterFactory>*>(
               opt_address);
-      *value = ptr->get() ? ptr->get()->Name() : "nullptr";
+      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
       break;
     }
     case OptionType::kMemTableRepFactory: {
       const auto* ptr =
           reinterpret_cast<const std::shared_ptr<MemTableRepFactory>*>(
               opt_address);
-      *value = ptr->get() ? ptr->get()->Name() : "nullptr";
+      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
       break;
     }
     case OptionType::kMergeOperator: {
       const auto* ptr =
           reinterpret_cast<const std::shared_ptr<MergeOperator>*>(opt_address);
-      *value = ptr->get() ? ptr->get()->Name() : "nullptr";
+      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
+      break;
+    }
+    case OptionType::kFilterPolicy: {
+      const auto* ptr =
+          reinterpret_cast<const std::shared_ptr<FilterPolicy>*>(opt_address);
+      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
+      break;
+    }
+    case OptionType::kChecksumType:
+      return SerializeChecksumType(
+          *reinterpret_cast<const ChecksumType*>(opt_address), value);
+    case OptionType::kBlockBasedTableIndexType:
+      return SerializeBlockBasedTableIndexType(
+          *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(
+              opt_address),
+          value);
+    case OptionType::kFlushBlockPolicyFactory: {
+      const auto* ptr =
+          reinterpret_cast<const std::shared_ptr<FlushBlockPolicyFactory>*>(
+              opt_address);
+      *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
       break;
     }
     default:
@@ -881,6 +937,59 @@ Status GetStringFromColumnFamilyOptions(std::string* opt_string,
   return Status::OK();
 }
 
+bool SerializeSingleBlockBasedTableOption(
+    std::string* opt_string, const BlockBasedTableOptions& bbt_options,
+    const std::string& name, const std::string& delimiter) {
+  auto iter = block_based_table_type_info.find(name);
+  if (iter == block_based_table_type_info.end()) {
+    return false;
+  }
+  auto& opt_info = iter->second;
+  const char* opt_address =
+      reinterpret_cast<const char*>(&bbt_options) + opt_info.offset;
+  std::string value;
+  bool result = SerializeSingleOptionHelper(opt_address, opt_info.type, &value);
+  if (result) {
+    *opt_string = name + "=" + value + delimiter;
+  }
+  return result;
+}
+
+Status GetStringFromBlockBasedTableOptions(
+    std::string* opt_string, const BlockBasedTableOptions& bbt_options,
+    const std::string& delimiter) {
+  assert(opt_string);
+  opt_string->clear();
+  for (auto iter = block_based_table_type_info.begin();
+       iter != block_based_table_type_info.end(); ++iter) {
+    if (iter->second.verification == OptionVerificationType::kDeprecated) {
+      // If the option is no longer used in rocksdb and marked as deprecated,
+      // we skip it in the serialization.
+      continue;
+    }
+    std::string single_output;
+    bool result = SerializeSingleBlockBasedTableOption(
+        &single_output, bbt_options, iter->first, delimiter);
+    assert(result);
+    if (result) {
+      opt_string->append(single_output);
+    }
+  }
+  return Status::OK();
+}
+
+Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf,
+                                 const std::string& delimiter) {
+  const auto* bbtf = dynamic_cast<const BlockBasedTableFactory*>(tf);
+  opts_str->clear();
+  if (bbtf != nullptr) {
+    return GetStringFromBlockBasedTableOptions(
+        opts_str, bbtf->GetTableOptions(), delimiter);
+  }
+
+  return Status::OK();
+}
+
 bool ParseDBOption(const std::string& name, const std::string& org_value,
                    DBOptions* new_options, bool input_string_escaped = false) {
   const std::string& value =
@@ -908,67 +1017,73 @@ bool ParseDBOption(const std::string& name, const std::string& org_value,
   return true;
 }
 
+std::string ParseBlockBasedTableOption(const std::string& name,
+                                       const std::string& org_value,
+                                       BlockBasedTableOptions* new_options,
+                                       bool input_string_escaped = false) {
+  const std::string& value =
+      input_string_escaped ? UnescapeOptionString(org_value) : org_value;
+  if (!input_string_escaped) {
+    // if the input string is not escaped, it means this function is
+    // invoked from SetOptions, which takes the old format.
+    if (name == "block_cache") {
+      new_options->block_cache = NewLRUCache(ParseSizeT(value));
+      return "";
+    } else if (name == "block_cache_compressed") {
+      new_options->block_cache_compressed = NewLRUCache(ParseSizeT(value));
+      return "";
+    } else if (name == "filter_policy") {
+      // Expect the following format
+      // bloomfilter:int:bool
+      const std::string kName = "bloomfilter:";
+      if (value.compare(0, kName.size(), kName) != 0) {
+        return "Invalid filter policy name";
+      }
+      size_t pos = value.find(':', kName.size());
+      if (pos == std::string::npos) {
+        return "Invalid filter policy config, missing bits_per_key";
+      }
+      int bits_per_key =
+          ParseInt(trim(value.substr(kName.size(), pos - kName.size())));
+      bool use_block_based_builder =
+          ParseBoolean("use_block_based_builder", trim(value.substr(pos + 1)));
+      new_options->filter_policy.reset(
+          NewBloomFilterPolicy(bits_per_key, use_block_based_builder));
+      return "";
+    }
+  }
+  const auto iter = block_based_table_type_info.find(name);
+  if (iter == block_based_table_type_info.end()) {
+    return "Unrecognized option";
+  }
+  const auto& opt_info = iter->second;
+  if (!ParseOptionHelper(reinterpret_cast<char*>(new_options) + opt_info.offset,
+                         opt_info.type, value)) {
+    return "Invalid value";
+  }
+  return "";
+}
+
 Status GetBlockBasedTableOptionsFromMap(
     const BlockBasedTableOptions& table_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    BlockBasedTableOptions* new_table_options) {
+    BlockBasedTableOptions* new_table_options, bool input_strings_escaped) {
-
   assert(new_table_options);
   *new_table_options = table_options;
   for (const auto& o : opts_map) {
-    try {
+    auto error_message = ParseBlockBasedTableOption(
-      if (o.first == "cache_index_and_filter_blocks") {
+        o.first, o.second, new_table_options, input_strings_escaped);
-        new_table_options->cache_index_and_filter_blocks =
+    if (error_message != "") {
-          ParseBoolean(o.first, o.second);
+      const auto iter = block_based_table_type_info.find(o.first);
-      } else if (o.first == "index_type") {
+      if (iter == block_based_table_type_info.end() ||
-        new_table_options->index_type = ParseBlockBasedTableIndexType(o.second);
+          !input_strings_escaped ||  // !input_strings_escaped indicates
-      } else if (o.first == "hash_index_allow_collision") {
+                                     // the old API, where everything is
-        new_table_options->hash_index_allow_collision =
+                                     // parsable.
-          ParseBoolean(o.first, o.second);
+          (iter->second.verification != OptionVerificationType::kByName &&
-      } else if (o.first == "checksum") {
+           iter->second.verification != OptionVerificationType::kDeprecated)) {
-        new_table_options->checksum =
+        return Status::InvalidArgument("Can't parse BlockBasedTableOptions:",
-          ParseBlockBasedTableChecksumType(o.second);
+                                       o.first + " " + error_message);
-      } else if (o.first == "no_block_cache") {
-        new_table_options->no_block_cache = ParseBoolean(o.first, o.second);
-      } else if (o.first == "block_cache") {
-        new_table_options->block_cache = NewLRUCache(ParseSizeT(o.second));
-      } else if (o.first == "block_cache_compressed") {
-        new_table_options->block_cache_compressed =
-          NewLRUCache(ParseSizeT(o.second));
-      } else if (o.first == "block_size") {
-        new_table_options->block_size = ParseSizeT(o.second);
-      } else if (o.first == "block_size_deviation") {
-        new_table_options->block_size_deviation = ParseInt(o.second);
-      } else if (o.first == "block_restart_interval") {
-        new_table_options->block_restart_interval = ParseInt(o.second);
-      } else if (o.first == "filter_policy") {
-        // Expect the following format
-        // bloomfilter:int:bool
-        const std::string kName = "bloomfilter:";
-        if (o.second.compare(0, kName.size(), kName) != 0) {
-          return Status::InvalidArgument("Invalid filter policy name");
-        }
-        size_t pos = o.second.find(':', kName.size());
-        if (pos == std::string::npos) {
-          return Status::InvalidArgument("Invalid filter policy config, "
-                                         "missing bits_per_key");
-        }
-        int bits_per_key = ParseInt(
-            trim(o.second.substr(kName.size(), pos - kName.size())));
-        bool use_block_based_builder =
-          ParseBoolean("use_block_based_builder",
-                       trim(o.second.substr(pos + 1)));
-        new_table_options->filter_policy.reset(
-            NewBloomFilterPolicy(bits_per_key, use_block_based_builder));
-      } else if (o.first == "whole_key_filtering") {
-        new_table_options->whole_key_filtering =
-          ParseBoolean(o.first, o.second);
-      } else {
-        return Status::InvalidArgument("Unrecognized option: " + o.first);
       }
-    } catch (std::exception& e) {
-      return Status::InvalidArgument("error parsing " + o.first + ":" +
-                                     std::string(e.what()));
     }
   }
   return Status::OK();
@@ -1110,5 +1225,26 @@ Status GetOptionsFromString(const Options& base_options,
   return Status::OK();
 }
 
+Status GetTableFactoryFromMap(
+    const std::string& factory_name,
+    const std::unordered_map<std::string, std::string>& opt_map,
+    std::shared_ptr<TableFactory>* table_factory) {
+  Status s;
+  if (factory_name == BlockBasedTableFactory().Name()) {
+    BlockBasedTableOptions bbt_opt;
+    s = GetBlockBasedTableOptionsFromMap(BlockBasedTableOptions(), opt_map,
+                                         &bbt_opt, true);
+    if (!s.ok()) {
+      return s;
+    }
+    table_factory->reset(new BlockBasedTableFactory(bbt_opt));
+    return Status::OK();
+  }
+  // Return OK for not supported table factories as TableFactory
+  // Deserialization is optional.
+  table_factory->reset();
+  return Status::OK();
+}
+
 #endif  // !ROCKSDB_LITE
 }  // namespace rocksdb

util/options_helper.h

@@ -9,6 +9,7 @@
 #include <stdexcept>
 #include "rocksdb/options.h"
 #include "rocksdb/status.h"
+#include "rocksdb/table.h"
 #include "util/mutable_cf_options.h"
 
 #ifndef ROCKSDB_LITE
@@ -55,6 +56,14 @@ Status GetMutableOptionsFromStrings(
     const std::unordered_map<std::string, std::string>& options_map,
     MutableCFOptions* new_options);
 
+Status GetTableFactoryFromMap(
+    const std::string& factory_name,
+    const std::unordered_map<std::string, std::string>& opt_map,
+    std::shared_ptr<TableFactory>* table_factory);
+
+Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf,
+                                 const std::string& delimiter = ";  ");
+
 enum class OptionType {
   kBoolean,
   kInt,
@@ -74,6 +83,10 @@ enum class OptionType {
   kCompactionFilterFactory,
   kMergeOperator,
   kMemTableRepFactory,
+  kBlockBasedTableIndexType,
+  kFilterPolicy,
+  kFlushBlockPolicyFactory,
+  kChecksumType,
   kUnknown
 };
 
@@ -401,6 +414,48 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
      {offsetof(struct ColumnFamilyOptions, compaction_style),
       OptionType::kCompactionStyle, OptionVerificationType::kNormal}}};
 
+static std::unordered_map<std::string,
+                          OptionTypeInfo> block_based_table_type_info = {
+    /* currently not supported
+      std::shared_ptr<Cache> block_cache = nullptr;
+      std::shared_ptr<Cache> block_cache_compressed = nullptr;
+     */
+    {"flush_block_policy_factory",
+     {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
+      OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName}},
+    {"cache_index_and_filter_blocks",
+     {offsetof(struct BlockBasedTableOptions, cache_index_and_filter_blocks),
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
+    {"index_type",
+     {offsetof(struct BlockBasedTableOptions, index_type),
+      OptionType::kBlockBasedTableIndexType, OptionVerificationType::kNormal}},
+    {"hash_index_allow_collision",
+     {offsetof(struct BlockBasedTableOptions, hash_index_allow_collision),
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
+    {"checksum",
+     {offsetof(struct BlockBasedTableOptions, checksum),
+      OptionType::kChecksumType, OptionVerificationType::kNormal}},
+    {"no_block_cache",
+     {offsetof(struct BlockBasedTableOptions, no_block_cache),
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
+    {"block_size",
+     {offsetof(struct BlockBasedTableOptions, block_size), OptionType::kSizeT,
+      OptionVerificationType::kNormal}},
+    {"block_size_deviation",
+     {offsetof(struct BlockBasedTableOptions, block_size_deviation),
+      OptionType::kInt, OptionVerificationType::kNormal}},
+    {"block_restart_interval",
+     {offsetof(struct BlockBasedTableOptions, block_restart_interval),
+      OptionType::kInt, OptionVerificationType::kNormal}},
+    {"filter_policy",
+     {offsetof(struct BlockBasedTableOptions, filter_policy),
+      OptionType::kFilterPolicy, OptionVerificationType::kByName}},
+    {"whole_key_filtering",
+     {offsetof(struct BlockBasedTableOptions, whole_key_filtering),
+      OptionType::kBoolean, OptionVerificationType::kNormal}},
+    {"format_version",
+     {offsetof(struct BlockBasedTableOptions, format_version),
+      OptionType::kUInt32T, OptionVerificationType::kNormal}}};
 }  // namespace rocksdb
 
 #endif  // !ROCKSDB_LITE

util/options_parser.cc

@@ -66,6 +66,7 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
   writable->Append(options_file_content + "\n");
 
   for (size_t i = 0; i < cf_opts.size(); ++i) {
+    // CFOptions section
     writable->Append("\n[" + opt_section_titles[kOptionSectionCFOptions] +
                      " \"" + EscapeOptionString(cf_names[i]) + "\"]\n  ");
     s = GetStringFromColumnFamilyOptions(&options_file_content, cf_opts[i],
@@ -75,6 +76,18 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
       return s;
     }
     writable->Append(options_file_content + "\n");
+    // TableOptions section
+    auto* tf = cf_opts[i].table_factory.get();
+    if (tf != nullptr) {
+      writable->Append("[" + opt_section_titles[kOptionSectionTableOptions] +
+                       tf->Name() + " \"" + EscapeOptionString(cf_names[i]) +
+                       "\"]\n  ");
+      s = GetStringFromTableFactory(&options_file_content, tf, "\n  ");
+      if (!s.ok()) {
+        return s;
+      }
+      writable->Append(options_file_content + "\n");
+    }
   }
   writable->Flush();
   writable->Fsync();
@@ -112,11 +125,11 @@ bool RocksDBOptionsParser::IsSection(const std::string& line) {
 }
 
 Status RocksDBOptionsParser::ParseSection(OptionSection* section,
+                                          std::string* title,
                                           std::string* argument,
                                           const std::string& line,
                                           const int line_num) {
   *section = kOptionSectionUnknown;
-  std::string sec_string;
   // A section is of the form [<SectionName> "<SectionArg>"], where
   // "<SectionArg>" is optional.
   size_t arg_start_pos = line.find("\"");
@@ -124,17 +137,30 @@ Status RocksDBOptionsParser::ParseSection(OptionSection* section,
   // The following if-then check tries to identify whether the input
   // section has the optional section argument.
   if (arg_start_pos != std::string::npos && arg_start_pos != arg_end_pos) {
-    sec_string = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true);
+    *title = TrimAndRemoveComment(line.substr(1, arg_start_pos - 1), true);
     *argument = UnescapeOptionString(
         line.substr(arg_start_pos + 1, arg_end_pos - arg_start_pos - 1));
   } else {
-    sec_string = TrimAndRemoveComment(line.substr(1, line.size() - 2), true);
+    *title = TrimAndRemoveComment(line.substr(1, line.size() - 2), true);
     *argument = "";
   }
   for (int i = 0; i < kOptionSectionUnknown; ++i) {
-    if (opt_section_titles[i] == sec_string) {
+    if (title->find(opt_section_titles[i]) == 0) {
-      *section = static_cast<OptionSection>(i);
+      if (i == kOptionSectionVersion || i == kOptionSectionDBOptions ||
-      return CheckSection(*section, *argument, line_num);
+          i == kOptionSectionCFOptions) {
+        if (title->size() == opt_section_titles[i].size()) {
+          // if true, then it indicats equal
+          *section = static_cast<OptionSection>(i);
+          return CheckSection(*section, *argument, line_num);
+        }
+      } else if (i == kOptionSectionTableOptions) {
+        // This type of sections has a sufffix at the end of the
+        // section title
+        if (title->size() > opt_section_titles[i].size()) {
+          *section = static_cast<OptionSection>(i);
+          return CheckSection(*section, *argument, line_num);
+        }
+      }
     }
   }
   return Status::InvalidArgument(std::string("Unknown section ") + line);
@@ -215,6 +241,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env) {
   }
 
   OptionSection section = kOptionSectionUnknown;
+  std::string title;
   std::string argument;
   std::unordered_map<std::string, std::string> opt_map;
   std::istringstream iss;
@@ -231,12 +258,12 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env) {
       continue;
     }
     if (IsSection(line)) {
-      s = EndSection(section, argument, opt_map);
+      s = EndSection(section, title, argument, opt_map);
       opt_map.clear();
       if (!s.ok()) {
         return s;
       }
-      s = ParseSection(&section, &argument, line, line_num);
+      s = ParseSection(&section, &title, &argument, line, line_num);
       if (!s.ok()) {
         return s;
       }
@@ -251,7 +278,7 @@ Status RocksDBOptionsParser::Parse(const std::string& file_name, Env* env) {
     }
   }
 
-  s = EndSection(section, argument, opt_map);
+  s = EndSection(section, title, argument, opt_map);
   opt_map.clear();
   if (!s.ok()) {
     return s;
@@ -280,13 +307,21 @@ Status RocksDBOptionsParser::CheckSection(const OptionSection section,
       return InvalidArgument(
           line_num,
           "Default column family must be the first CFOptions section "
-          "in the option config file");
+          "in the optio/n config file");
     } else if (GetCFOptions(section_arg) != nullptr) {
       return InvalidArgument(
           line_num,
           "Two identical column families found in option config file");
     }
     has_default_cf_options_ |= is_default_cf;
+  } else if (section == kOptionSectionTableOptions) {
+    if (GetCFOptions(section_arg) == nullptr) {
+      return InvalidArgument(
+          line_num, std::string(
+                        "Does not find a matched column family name in "
+                        "TableOptions section.  Column Family Name:") +
+                        section_arg);
+    }
   } else if (section == kOptionSectionVersion) {
     if (has_version_section_) {
       return InvalidArgument(
@@ -350,7 +385,8 @@ Status RocksDBOptionsParser::ParseVersionNumber(const std::string& ver_name,
 }
 
 Status RocksDBOptionsParser::EndSection(
-    const OptionSection section, const std::string& section_arg,
+    const OptionSection section, const std::string& section_title,
+    const std::string& section_arg,
     const std::unordered_map<std::string, std::string>& opt_map) {
   Status s;
   if (section == kOptionSectionDBOptions) {
@@ -372,6 +408,23 @@ Status RocksDBOptionsParser::EndSection(
     }
     // keep the parsed string.
     cf_opt_maps_.emplace_back(opt_map);
+  } else if (section == kOptionSectionTableOptions) {
+    assert(GetCFOptions(section_arg) != nullptr);
+    auto* cf_opt = GetCFOptionsImpl(section_arg);
+    if (cf_opt == nullptr) {
+      return Status::InvalidArgument(
+          "The specified column family must be defined before the "
+          "TableOptions section:",
+          section_arg);
+    }
+    // Ignore error as table factory deserialization is optional
+    s = GetTableFactoryFromMap(
+        section_title.substr(
+            opt_section_titles[kOptionSectionTableOptions].size()),
+        opt_map, &(cf_opt->table_factory));
+    if (!s.ok()) {
+      return s;
+    }
   } else if (section == kOptionSectionVersion) {
     for (const auto pair : opt_map) {
       if (pair.first == "rocksdb_version") {
@@ -493,6 +546,14 @@ bool AreEqualOptions(
           reinterpret_cast<const std::vector<CompressionType>*>(offset2);
       return (*vec1 == *vec2);
     }
+    case OptionType::kChecksumType:
+      return (*reinterpret_cast<const ChecksumType*>(offset1) ==
+              *reinterpret_cast<const ChecksumType*>(offset2));
+    case OptionType::kBlockBasedTableIndexType:
+      return (
+          *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(
+              offset1) ==
+          *reinterpret_cast<const BlockBasedTableOptions::IndexType*>(offset2));
     default:
       if (type_info.verification == OptionVerificationType::kByName) {
         std::string value1;
@@ -561,6 +622,11 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
     if (!s.ok()) {
       return s;
     }
+    s = VerifyTableFactory(cf_opts[i].table_factory.get(),
+                           parser.cf_opts()->at(i).table_factory.get());
+    if (!s.ok()) {
+      return s;
+    }
   }
 
   return Status::OK();
@@ -607,6 +673,59 @@ Status RocksDBOptionsParser::VerifyCFOptions(
   }
   return Status::OK();
 }
+
+Status RocksDBOptionsParser::VerifyBlockBasedTableFactory(
+    const BlockBasedTableFactory* base_tf,
+    const BlockBasedTableFactory* file_tf) {
+  if ((base_tf != nullptr) != (file_tf != nullptr)) {
+    return Status::Corruption(
+        "[RocksDBOptionsParser]: Inconsistent TableFactory class type");
+  }
+  if (base_tf == nullptr) {
+    return Status::OK();
+  }
+
+  const auto& base_opt = base_tf->GetTableOptions();
+  const auto& file_opt = file_tf->GetTableOptions();
+
+  for (auto& pair : block_based_table_type_info) {
+    if (pair.second.verification == OptionVerificationType::kDeprecated) {
+      // We skip checking deprecated variables as they might
+      // contain random values since they might not be initialized
+      continue;
+    }
+    if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
+                         reinterpret_cast<const char*>(&file_opt), pair.second,
+                         pair.first, nullptr)) {
+      return Status::Corruption(
+          "[RocksDBOptionsParser]: "
+          "failed the verification on BlockBasedTableOptions::",
+          pair.first);
+    }
+  }
+  return Status::OK();
+}
+
+Status RocksDBOptionsParser::VerifyTableFactory(const TableFactory* base_tf,
+                                                const TableFactory* file_tf) {
+  if (base_tf && file_tf) {
+    if (base_tf->Name() != file_tf->Name()) {
+      return Status::Corruption(
+          "[RocksDBOptionsParser]: "
+          "failed the verification on TableFactory->Name()");
+    }
+    auto s = VerifyBlockBasedTableFactory(
+        dynamic_cast<const BlockBasedTableFactory*>(base_tf),
+        dynamic_cast<const BlockBasedTableFactory*>(file_tf));
+    if (!s.ok()) {
+      return s;
+    }
+    // TODO(yhchiang): add checks for other table factory types
+  } else {
+    // TODO(yhchiang): further support sanity check here
+  }
+  return Status::OK();
+}
 }  // namespace rocksdb
 
 #endif  // !ROCKSDB_LITE

util/options_parser.h

@@ -11,23 +11,25 @@
 
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
+#include "table/block_based_table_factory.h"
 
 namespace rocksdb {
 
 #ifndef ROCKSDB_LITE
 
 #define ROCKSDB_OPTION_FILE_MAJOR 1
-#define ROCKSDB_OPTION_FILE_MINOR 0
+#define ROCKSDB_OPTION_FILE_MINOR 1
 
 enum OptionSection : char {
   kOptionSectionVersion = 0,
   kOptionSectionDBOptions,
   kOptionSectionCFOptions,
+  kOptionSectionTableOptions,
   kOptionSectionUnknown
 };
 
-static const std::string opt_section_titles[] = {"Version", "DBOptions",
+static const std::string opt_section_titles[] = {
-                                                 "CFOptions", "Unknown"};
+    "Version", "DBOptions", "CFOptions", "TableOptions/", "Unknown"};
 
 Status PersistRocksDBOptions(const DBOptions& db_opt,
                              const std::vector<std::string>& cf_names,
@@ -55,14 +57,8 @@ class RocksDBOptionsParser {
     return &cf_opt_maps_;
   }
 
-  const ColumnFamilyOptions* GetCFOptions(const std::string& name) const {
+  const ColumnFamilyOptions* GetCFOptions(const std::string& name) {
-    assert(cf_names_.size() == cf_opts_.size());
+    return GetCFOptionsImpl(name);
-    for (size_t i = 0; i < cf_names_.size(); ++i) {
-      if (cf_names_[i] == name) {
-        return &cf_opts_[i];
-      }
-    }
-    return nullptr;
   }
   size_t NumColumnFamilies() { return cf_opts_.size(); }
 
@@ -81,12 +77,20 @@ class RocksDBOptionsParser {
       const std::unordered_map<std::string, std::string>* new_opt_map =
           nullptr);
 
+  static Status VerifyTableFactory(const TableFactory* base_tf,
+                                   const TableFactory* file_tf);
+
+  static Status VerifyBlockBasedTableFactory(
+      const BlockBasedTableFactory* base_tf,
+      const BlockBasedTableFactory* file_tf);
+
   static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);
 
  protected:
   bool IsSection(const std::string& line);
-  Status ParseSection(OptionSection* section, std::string* argument,
+  Status ParseSection(OptionSection* section, std::string* title,
-                      const std::string& line, const int line_num);
+                      std::string* argument, const std::string& line,
+                      const int line_num);
 
   Status CheckSection(const OptionSection section,
                       const std::string& section_arg, const int line_num);
@@ -95,7 +99,8 @@ class RocksDBOptionsParser {
                         const std::string& line, const int line_num);
 
   Status EndSection(
-      const OptionSection section, const std::string& section_arg,
+      const OptionSection section, const std::string& title,
+      const std::string& section_arg,
       const std::unordered_map<std::string, std::string>& opt_map);
 
   Status ValidityCheck();
@@ -106,6 +111,16 @@ class RocksDBOptionsParser {
                             const std::string& ver_string, const int max_count,
                             int* version);
 
+  ColumnFamilyOptions* GetCFOptionsImpl(const std::string& name) {
+    assert(cf_names_.size() == cf_opts_.size());
+    for (size_t i = 0; i < cf_names_.size(); ++i) {
+      if (cf_names_[i] == name) {
+        return &cf_opts_[i];
+      }
+    }
+    return nullptr;
+  }
+
  private:
   DBOptions db_opt_;
   std::unordered_map<std::string, std::string> db_opt_map_;

util/options_test.cc

@@ -1565,6 +1565,23 @@ TEST_F(OptionsParserTest, DumpAndParse) {
   }
 }
 
+TEST_F(OptionsParserTest, DifferentDefault) {
+  const std::string kOptionsFileName = "test-persisted-options.ini";
+
+  ColumnFamilyOptions cf_level_opts;
+  cf_level_opts.OptimizeLevelStyleCompaction();
+
+  ColumnFamilyOptions cf_univ_opts;
+  cf_univ_opts.OptimizeUniversalStyleCompaction();
+
+  ASSERT_OK(PersistRocksDBOptions(DBOptions(), {"default", "universal"},
+                                  {cf_level_opts, cf_univ_opts},
+                                  kOptionsFileName, env_.get()));
+
+  RocksDBOptionsParser parser;
+  ASSERT_OK(parser.Parse(kOptionsFileName, env_.get()));
+}
+
 namespace {
 bool IsEscapedString(const std::string& str) {
   for (size_t i = 0; i < str.size(); ++i) {