[RocksDB Options] Support more options in RocksDBOptionParser for sanity check.

Summary: RocksDBOptionsParser now supports CompressionType and the following pointer-typed options in RocksDBOptionParser for sanity check: prefix_extractor table_factory comparator compaction_filter compaction_filter_factory merge_operator memtable_factory In the RocksDB Options file, only high level information about pointer-typed options are serialized, and those information is only used for verification / sanity check purpose. Test Plan: added more tests in options_test Reviewers: igor, IslamAbdelRahman, sdong, anthony Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47925
9 years ago · 5c7bf56d35
parent 115427ef63
commit 5c7bf56d35
6 changed files with 623 additions and 108 deletions
--- a/util/options_helper.cc
+++ b/util/options_helper.cc
@ -8,9 +8,13 @@
 #include <cctype>
 #include <cstdlib>
 #include <unordered_set>
+#include <vector>
 #include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/filter_policy.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/merge_operator.h"
 #include "rocksdb/options.h"
 #include "rocksdb/rate_limiter.h"
 #include "rocksdb/slice_transform.h"
@ -85,25 +89,89 @@ std::string UnescapeOptionString(const std::string& escaped_string) {
 }

 namespace {
-CompressionType ParseCompressionType(const std::string& type) {
-  if (type == "kNoCompression") {
-    return kNoCompression;
-  } else if (type == "kSnappyCompression") {
-    return kSnappyCompression;
-  } else if (type == "kZlibCompression") {
-    return kZlibCompression;
-  } else if (type == "kBZip2Compression") {
-    return kBZip2Compression;
-  } else if (type == "kLZ4Compression") {
-    return kLZ4Compression;
-  } else if (type == "kLZ4HCCompression") {
-    return kLZ4HCCompression;
-  } else if (type == "kZSTDNotFinalCompression") {
-    return kZSTDNotFinalCompression;
+std::string trim(const std::string& str) {
+  if (str.empty()) return std::string();
+  size_t start = 0;
+  size_t end = str.size() - 1;
+  while (isspace(str[start]) != 0 && start <= end) {
+    ++start;
+  }
+  while (isspace(str[end]) != 0 && start <= end) {
+    --end;
+  }
+  if (start <= end) {
+    return str.substr(start, end - start + 1);
+  }
+  return std::string();
+}
+
+bool SerializeCompressionType(const CompressionType& type, std::string* value) {
+  switch (type) {
+    case kNoCompression:
+      *value = "kNoCompression";
+      return true;
+    case kSnappyCompression:
+      *value = "kSnappyCompression";
+      return true;
+    case kZlibCompression:
+      *value = "kZlibCompression";
+      return true;
+    case kBZip2Compression:
+      *value = "kBZip2Compression";
+      return true;
+    case kLZ4Compression:
+      *value = "kLZ4Compression";
+      return true;
+    case kLZ4HCCompression:
+      *value = "kLZ4HCCompression";
+      return true;
+    case kZSTDNotFinalCompression:
+      *value = "kZSTDNotFinalCompression";
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool SerializeVectorCompressionType(const std::vector<CompressionType>& types,
+                                    std::string* value) {
+  std::stringstream ss;
+  bool result;
+  for (size_t i = 0; i < types.size(); ++i) {
+    if (i > 0) {
+      ss << ':';
+    }
+    std::string string_type;
+    result = SerializeCompressionType(types[i], &string_type);
+    if (result == false) {
+      return result;
+    }
+    ss << string_type;
+  }
+  *value = ss.str();
+  return true;
+}
+
+bool ParseCompressionType(const std::string& string_value,
+                          CompressionType* type) {
+  if (string_value == "kNoCompression") {
+    *type = kNoCompression;
+  } else if (string_value == "kSnappyCompression") {
+    *type = kSnappyCompression;
+  } else if (string_value == "kZlibCompression") {
+    *type = kZlibCompression;
+  } else if (string_value == "kBZip2Compression") {
+    *type = kBZip2Compression;
+  } else if (string_value == "kLZ4Compression") {
+    *type = kLZ4Compression;
+  } else if (string_value == "kLZ4HCCompression") {
+    *type = kLZ4HCCompression;
+  } else if (string_value == "kZSTDNotFinalCompression") {
+    *type = kZSTDNotFinalCompression;
  } else {
-    throw std::invalid_argument("Unknown compression type: " + type);
+    return false;
  }
-  return kNoCompression;
+  return true;
 }

 BlockBasedTableOptions::IndexType ParseBlockBasedTableIndexType(
@ -205,7 +273,6 @@ double ParseDouble(const std::string& value) {
  return std::strtod(value.c_str(), 0);
 #endif
 }
-
 static const std::unordered_map<char, std::string>
    compaction_style_to_string_map = {
        {kCompactionStyleLevel, "kCompactionStyleLevel"},
@ -229,6 +296,83 @@ std::string CompactionStyleToString(const CompactionStyle style) {
  return iter->second;
 }

+bool ParseVectorCompressionType(
+    const std::string& value,
+    std::vector<CompressionType>* compression_per_level) {
+  compression_per_level->clear();
+  size_t start = 0;
+  while (start < value.size()) {
+    size_t end = value.find(':', start);
+    bool is_ok;
+    CompressionType type;
+    if (end == std::string::npos) {
+      is_ok = ParseCompressionType(value.substr(start), &type);
+      if (!is_ok) {
+        return false;
+      }
+      compression_per_level->emplace_back(type);
+      break;
+    } else {
+      is_ok = ParseCompressionType(value.substr(start, end - start), &type);
+      if (!is_ok) {
+        return false;
+      }
+      compression_per_level->emplace_back(type);
+      start = end + 1;
+    }
+  }
+  return true;
+}
+
+bool ParseSliceTransformHelper(
+    const std::string& kFixedPrefixName, const std::string& kCappedPrefixName,
+    const std::string& value,
+    std::shared_ptr<const SliceTransform>* slice_transform) {
+  auto& pe_value = value;
+  if (pe_value.size() > kFixedPrefixName.size() &&
+      pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
+    int prefix_length = ParseInt(trim(value.substr(kFixedPrefixName.size())));
+    slice_transform->reset(NewFixedPrefixTransform(prefix_length));
+  } else if (pe_value.size() > kCappedPrefixName.size() &&
+             pe_value.compare(0, kCappedPrefixName.size(), kCappedPrefixName) ==
+                 0) {
+    int prefix_length =
+        ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
+    slice_transform->reset(NewCappedPrefixTransform(prefix_length));
+  } else if (value == "nullptr") {
+    slice_transform->reset();
+  } else {
+    return false;
+  }
+
+  return true;
+}
+
+bool ParseSliceTransform(
+    const std::string& value,
+    std::shared_ptr<const SliceTransform>* slice_transform) {
+  // While we normally don't convert the string representation of a
+  // pointer-typed option into its instance, here we do so for backward
+  // compatibility as we allow this action in SetOption().
+
+  // TODO(yhchiang): A possible better place for these serialization /
+  // deserialization is inside the class definition of pointer-typed
+  // option itself, but this requires a bigger change of public API.
+  bool result =
+      ParseSliceTransformHelper("fixed:", "capped:", value, slice_transform);
+  if (result) {
+    return result;
+  }
+  result = ParseSliceTransformHelper(
+      "rocksdb.FixedPrefix.", "rocksdb.CappedPrefix.", value, slice_transform);
+  if (result) {
+    return result;
+  }
+  // TODO(yhchiang): we can further support other default
+  //                 SliceTransforms here.
+  return false;
+}
+
 bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
                       const std::string& value) {
  switch (opt_type) {
@ -260,12 +404,24 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
      *reinterpret_cast<CompactionStyle*>(opt_address) =
          ParseCompactionStyle(value);
      break;
+    case OptionType::kCompressionType:
+      return ParseCompressionType(
+          value, reinterpret_cast<CompressionType*>(opt_address));
+    case OptionType::kVectorCompressionType:
+      return ParseVectorCompressionType(
+          value, reinterpret_cast<std::vector<CompressionType>*>(opt_address));
+    case OptionType::kSliceTransform:
+      return ParseSliceTransform(
+          value, reinterpret_cast<std::shared_ptr<const SliceTransform>*>(
+                     opt_address));
    default:
      return false;
  }
  return true;
 }

+}  // anonymouse namespace
+
 bool SerializeSingleOptionHelper(const char* opt_address,
                                 const OptionType opt_type,
                                 std::string* value) {
@ -300,13 +456,69 @@ bool SerializeSingleOptionHelper(const char* opt_address,
      *value = CompactionStyleToString(
          *(reinterpret_cast<const CompactionStyle*>(opt_address)));
      break;
+    case OptionType::kCompressionType:
+      return SerializeCompressionType(
+          *(reinterpret_cast<const CompressionType*>(opt_address)), value);
+    case OptionType::kVectorCompressionType:
+      return SerializeVectorCompressionType(
+          *(reinterpret_cast<const std::vector<CompressionType>*>(opt_address)),
+          value);
+      break;
+    case OptionType::kSliceTransform: {
+      const auto* slice_transform_ptr =
+          reinterpret_cast<const std::shared_ptr<const SliceTransform>*>(
+              opt_address);
+      *value = slice_transform_ptr->get() ? slice_transform_ptr->get()->Name()
+                                          : "nullptr";
+      break;
+    }
+    case OptionType::kTableFactory: {
+      const auto* table_factory_ptr =
+          reinterpret_cast<const std::shared_ptr<const TableFactory>*>(
+              opt_address);
+      *value = table_factory_ptr->get() ? table_factory_ptr->get()->Name()
+                                        : "nullptr";
+      break;
+    }
+    case OptionType::kComparator: {
+      // it's a const pointer of const Comparator*
+      const auto* ptr = reinterpret_cast<const Comparator* const*>(opt_address);
+      *value = *ptr ? (*ptr)->Name() : "nullptr";
+      break;
+    }
+    case OptionType::kCompactionFilter: {
+      // it's a const pointer of const CompactionFilter*
+      const auto* ptr =
+          reinterpret_cast<const CompactionFilter* const*>(opt_address);
+      *value = *ptr ? (*ptr)->Name() : "nullptr";
+      break;
+    }
+    case OptionType::kCompactionFilterFactory: {
+      const auto* ptr =
+          reinterpret_cast<const std::shared_ptr<CompactionFilterFactory>*>(
+              opt_address);
+      *value = ptr->get() ? ptr->get()->Name() : "nullptr";
+      break;
+    }
+    case OptionType::kMemTableRepFactory: {
+      const auto* ptr =
+          reinterpret_cast<const std::shared_ptr<MemTableRepFactory>*>(
+              opt_address);
+      *value = ptr->get() ? ptr->get()->Name() : "nullptr";
+      break;
+    }
+    case OptionType::kMergeOperator: {
+      const auto* ptr =
+          reinterpret_cast<const std::shared_ptr<MergeOperator>*>(opt_address);
+      *value = ptr->get() ? ptr->get()->Name() : "nullptr";
+      break;
+    }
    default:
      return false;
  }
  return true;
 }

-}  // anonymouse namespace

 template<typename OptionsType>
 bool ParseMemtableOptions(const std::string& name, const std::string& value,
@ -427,26 +639,6 @@ Status GetMutableOptionsFromStrings(
  return Status::OK();
 }

-namespace {
-
-std::string trim(const std::string& str) {
-  if (str.empty()) return std::string();
-  size_t start = 0;
-  size_t end = str.size() - 1;
-  while (isspace(str[start]) != 0 && start <= end) {
-    ++start;
-  }
-  while (isspace(str[end]) != 0 && start <= end) {
-    --end;
-  }
-  if (start <= end) {
-    return str.substr(start, end - start + 1);
-  }
-  return std::string();
-}
-
-}  // anonymous namespace
-
 Status StringToMap(const std::string& opts_str,
                   std::unordered_map<std::string, std::string>* opts_map) {
  assert(opts_map);
@ -559,23 +751,6 @@ bool ParseColumnFamilyOption(const std::string& name,
        return false;
      }
      new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt));
-    } else if (name == "compression") {
-      new_options->compression = ParseCompressionType(value);
-    } else if (name == "compression_per_level") {
-      new_options->compression_per_level.clear();
-      size_t start = 0;
-      while (true) {
-        size_t end = value.find(':', start);
-        if (end == std::string::npos) {
-          new_options->compression_per_level.push_back(
-              ParseCompressionType(value.substr(start)));
-          break;
-        } else {
-          new_options->compression_per_level.push_back(
-              ParseCompressionType(value.substr(start, end - start)));
-          start = end + 1;
-        }
-      }
    } else if (name == "compression_opts") {
      size_t start = 0;
      size_t end = value.find(':');
@ -603,26 +778,6 @@ bool ParseColumnFamilyOption(const std::string& name,
    } else if (name == "compaction_options_fifo") {
      new_options->compaction_options_fifo.max_table_files_size =
          ParseUint64(value);
-    } else if (name == "prefix_extractor") {
-      const std::string kFixedPrefixName = "fixed:";
-      const std::string kCappedPrefixName = "capped:";
-      auto& pe_value = value;
-      if (pe_value.size() > kFixedPrefixName.size() &&
-          pe_value.compare(0, kFixedPrefixName.size(), kFixedPrefixName) == 0) {
-        int prefix_length =
-            ParseInt(trim(value.substr(kFixedPrefixName.size())));
-        new_options->prefix_extractor.reset(
-            NewFixedPrefixTransform(prefix_length));
-      } else if (pe_value.size() > kCappedPrefixName.size() &&
-                 pe_value.compare(0, kCappedPrefixName.size(),
-                                  kCappedPrefixName) == 0) {
-        int prefix_length =
-            ParseInt(trim(pe_value.substr(kCappedPrefixName.size())));
-        new_options->prefix_extractor.reset(
-            NewCappedPrefixTransform(prefix_length));
-      } else {
-        return false;
-      }
    } else {
      auto iter = cf_options_type_info.find(name);
      if (iter == cf_options_type_info.end()) {
@ -740,9 +895,12 @@ bool ParseDBOption(const std::string& name, const std::string& org_value,
        return false;
      }
      const auto& opt_info = iter->second;
+      if (opt_info.verification != OptionVerificationType::kByName &&
+          opt_info.verification != OptionVerificationType::kDeprecated) {
        return ParseOptionHelper(
-          reinterpret_cast<char*>(new_options) + opt_info.offset, opt_info.type,
-          value);
+            reinterpret_cast<char*>(new_options) + opt_info.offset,
+            opt_info.type, value);
+      }
    }
  } catch (const std::exception& e) {
    return false;
@ -881,9 +1039,14 @@ Status GetColumnFamilyOptionsFromMap(
  for (const auto& o : opts_map) {
    if (!ParseColumnFamilyOption(o.first, o.second, new_options,
                                 input_strings_escaped)) {
+      auto iter = cf_options_type_info.find(o.first);
+      if (iter == cf_options_type_info.end() ||
+          (iter->second.verification != OptionVerificationType::kByName &&
+           iter->second.verification != OptionVerificationType::kDeprecated)) {
        return Status::InvalidArgument("Can't parse option " + o.first);
      }
    }
+  }
  return Status::OK();
 }

@ -907,6 +1070,8 @@ Status GetDBOptionsFromMap(
  *new_options = base_options;
  for (const auto& o : opts_map) {
    if (!ParseDBOption(o.first, o.second, new_options, input_strings_escaped)) {
+      // Note that options with kDeprecated validation will pass ParseDBOption
+      // and will not hit the below statement.
      return Status::InvalidArgument("Can't parse option " + o.first);
    }
  }
--- a/util/options_helper.h
+++ b/util/options_helper.h
@ -65,11 +65,22 @@ enum class OptionType {
  kString,
  kDouble,
  kCompactionStyle,
+  kSliceTransform,
+  kCompressionType,
+  kVectorCompressionType,
+  kTableFactory,
+  kComparator,
+  kCompactionFilter,
+  kCompactionFilterFactory,
+  kMergeOperator,
+  kMemTableRepFactory,
  kUnknown
 };

 enum class OptionVerificationType {
  kNormal,
+  kByName,     // The option is pointer typed so we can only verify
+               // based on it's name.
  kDeprecated  // The option is no longer used in rocksdb. The RocksDB
               // OptionsParser will still accept this option if it
               // happen to exists in some Options file.  However, the
@ -85,6 +96,11 @@ struct OptionTypeInfo {
  OptionVerificationType verification;
 };

+// A helper function that converts "opt_address" to a std::string
+// based on the specified OptionType.
+bool SerializeSingleOptionHelper(const char* opt_address,
+                                 const OptionType opt_type, std::string* value);
+
 static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
    /*
     // not yet supported
@ -226,7 +242,6 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
    CompactionOptionsFIFO compaction_options_fifo;
    CompactionOptionsUniversal compaction_options_universal;
    CompressionOptions compression_opts;
-    CompressionType compression;
    TablePropertiesCollectorFactories table_properties_collector_factories;
    typedef std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>
        TablePropertiesCollectorFactories;
@ -234,14 +249,6 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
                                     uint34_t* existing_value_size,
                                     Slice delta_value,
                                     std::string* merged_value);
-    const CompactionFilter* compaction_filter;
-    const Comparator* comparator;
-    std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
-    std::shared_ptr<MemTableRepFactory> memtable_factory;
-    std::shared_ptr<MergeOperator> merge_operator;
-    std::shared_ptr<TableFactory> table_factory;
-    std::shared_ptr<const SliceTransform> prefix_extractor;
-    std::vector<CompressionType> compression_per_level;
    std::vector<int> max_bytes_for_level_multiplier_additional;
     */
    {"compaction_measure_io_stats",
@ -360,6 +367,33 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
    {"rate_limit_delay_max_milliseconds",
     {offsetof(struct ColumnFamilyOptions, rate_limit_delay_max_milliseconds),
      OptionType::kUInt, OptionVerificationType::kDeprecated}},
+    {"compression",
+     {offsetof(struct ColumnFamilyOptions, compression),
+      OptionType::kCompressionType, OptionVerificationType::kNormal}},
+    {"compression_per_level",
+     {offsetof(struct ColumnFamilyOptions, compression_per_level),
+      OptionType::kVectorCompressionType, OptionVerificationType::kNormal}},
+    {"comparator",
+     {offsetof(struct ColumnFamilyOptions, comparator), OptionType::kComparator,
+      OptionVerificationType::kByName}},
+    {"prefix_extractor",
+     {offsetof(struct ColumnFamilyOptions, prefix_extractor),
+      OptionType::kSliceTransform, OptionVerificationType::kByName}},
+    {"memtable_factory",
+     {offsetof(struct ColumnFamilyOptions, memtable_factory),
+      OptionType::kMemTableRepFactory, OptionVerificationType::kByName}},
+    {"table_factory",
+     {offsetof(struct ColumnFamilyOptions, table_factory),
+      OptionType::kTableFactory, OptionVerificationType::kByName}},
+    {"compaction_filter",
+     {offsetof(struct ColumnFamilyOptions, compaction_filter),
+      OptionType::kCompactionFilter, OptionVerificationType::kByName}},
+    {"compaction_filter_factory",
+     {offsetof(struct ColumnFamilyOptions, compaction_filter_factory),
+      OptionType::kCompactionFilterFactory, OptionVerificationType::kByName}},
+    {"merge_operator",
+     {offsetof(struct ColumnFamilyOptions, merge_operator),
+      OptionType::kMergeOperator, OptionVerificationType::kByName}},
    {"compaction_style",
     {offsetof(struct ColumnFamilyOptions, compaction_style),
      OptionType::kCompactionStyle, OptionVerificationType::kNormal}}};
--- a/util/options_parser.cc
+++ b/util/options_parser.cc
@ -88,8 +88,10 @@ RocksDBOptionsParser::RocksDBOptionsParser() { Reset(); }

 void RocksDBOptionsParser::Reset() {
  db_opt_ = DBOptions();
+  db_opt_map_.clear();
  cf_names_.clear();
  cf_opts_.clear();
+  cf_opt_maps_.clear();
  has_version_section_ = false;
  has_db_options_ = false;
  has_default_cf_options_ = false;
@ -356,6 +358,7 @@ Status RocksDBOptionsParser::EndSection(
    if (!s.ok()) {
      return s;
    }
+    db_opt_map_ = opt_map;
  } else if (section == kOptionSectionCFOptions) {
    // This condition should be ensured earlier in ParseSection
    // so we make an assertion here.
@ -367,6 +370,8 @@ Status RocksDBOptionsParser::EndSection(
    if (!s.ok()) {
      return s;
    }
+    // keep the parsed string.
+    cf_opt_maps_.emplace_back(opt_map);
  } else if (section == kOptionSectionVersion) {
    for (const auto pair : opt_map) {
      if (pair.first == "rocksdb_version") {
@ -444,8 +449,10 @@ bool AreEqualDoubles(const double a, const double b) {
  return (fabs(a - b) < 0.00001);
 }

-bool AreEqualOptions(const char* opt1, const char* opt2,
-                     const OptionTypeInfo& type_info) {
+bool AreEqualOptions(
+    const char* opt1, const char* opt2, const OptionTypeInfo& type_info,
+    const std::string& opt_name,
+    const std::unordered_map<std::string, std::string>* opt_map) {
  const char* offset1 = opt1 + type_info.offset;
  const char* offset2 = opt2 + type_info.offset;
  switch (type_info.type) {
@ -476,7 +483,34 @@ bool AreEqualOptions(const char* opt1, const char* opt2,
    case OptionType::kCompactionStyle:
      return (*reinterpret_cast<const CompactionStyle*>(offset1) ==
              *reinterpret_cast<const CompactionStyle*>(offset2));
+    case OptionType::kCompressionType:
+      return (*reinterpret_cast<const CompressionType*>(offset1) ==
+              *reinterpret_cast<const CompressionType*>(offset2));
+    case OptionType::kVectorCompressionType: {
+      const auto* vec1 =
+          reinterpret_cast<const std::vector<CompressionType>*>(offset1);
+      const auto* vec2 =
+          reinterpret_cast<const std::vector<CompressionType>*>(offset2);
+      return (*vec1 == *vec2);
+    }
    default:
+      if (type_info.verification == OptionVerificationType::kByName) {
+        std::string value1;
+        bool result =
+            SerializeSingleOptionHelper(offset1, type_info.type, &value1);
+        if (result == false) {
+          return false;
+        }
+        if (opt_map == nullptr) {
+          return true;
+        }
+        auto iter = opt_map->find(opt_name);
+        if (iter == opt_map->end()) {
+          return true;
+        } else {
+          return (value1 == iter->second);
+        }
+      }
      return false;
  }
 }
@ -495,7 +529,7 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
  }

  // Verify DBOptions
-  s = VerifyDBOptions(db_opt, *parser.db_opt());
+  s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map());
  if (!s.ok()) {
    return s;
  }
@ -522,7 +556,8 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
        "the same number of column families as the db instance.");
  }
  for (size_t i = 0; i < cf_opts.size(); ++i) {
-    s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i));
+    s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i),
+                        &(parser.cf_opt_maps()->at(i)));
    if (!s.ok()) {
      return s;
    }
@ -531,8 +566,9 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
  return Status::OK();
 }

-Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
-                                             const DBOptions& new_opt) {
+Status RocksDBOptionsParser::VerifyDBOptions(
+    const DBOptions& base_opt, const DBOptions& new_opt,
+    const std::unordered_map<std::string, std::string>* opt_map) {
  for (auto pair : db_options_type_info) {
    if (pair.second.verification == OptionVerificationType::kDeprecated) {
      // We skip checking deprecated variables as they might
@ -540,8 +576,8 @@ Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
      continue;
    }
    if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
-                         reinterpret_cast<const char*>(&new_opt),
-                         pair.second)) {
+                         reinterpret_cast<const char*>(&new_opt), pair.second,
+                         pair.first, nullptr)) {
      return Status::Corruption(
          "[RocksDBOptionsParser]: "
          "failed the verification on DBOptions::",
@ -552,7 +588,8 @@ Status RocksDBOptionsParser::VerifyDBOptions(const DBOptions& base_opt,
 }

 Status RocksDBOptionsParser::VerifyCFOptions(
-    const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt) {
+    const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt,
+    const std::unordered_map<std::string, std::string>* new_opt_map) {
  for (auto& pair : cf_options_type_info) {
    if (pair.second.verification == OptionVerificationType::kDeprecated) {
      // We skip checking deprecated variables as they might
@ -560,8 +597,8 @@ Status RocksDBOptionsParser::VerifyCFOptions(
      continue;
    }
    if (!AreEqualOptions(reinterpret_cast<const char*>(&base_opt),
-                         reinterpret_cast<const char*>(&new_opt),
-                         pair.second)) {
+                         reinterpret_cast<const char*>(&new_opt), pair.second,
+                         pair.first, new_opt_map)) {
      return Status::Corruption(
          "[RocksDBOptionsParser]: "
          "failed the verification on ColumnFamilyOptions::",
--- a/util/options_parser.h
+++ b/util/options_parser.h
@ -45,8 +45,15 @@ class RocksDBOptionsParser {
                                          const bool trim_only = false);

  const DBOptions* db_opt() const { return &db_opt_; }
+  const std::unordered_map<std::string, std::string>* db_opt_map() const {
+    return &db_opt_map_;
+  }
  const std::vector<ColumnFamilyOptions>* cf_opts() const { return &cf_opts_; }
  const std::vector<std::string>* cf_names() const { return &cf_names_; }
+  const std::vector<std::unordered_map<std::string, std::string>>* cf_opt_maps()
+      const {
+    return &cf_opt_maps_;
+  }

  const ColumnFamilyOptions* GetCFOptions(const std::string& name) const {
    assert(cf_names_.size() == cf_opts_.size());
@ -64,11 +71,15 @@ class RocksDBOptionsParser {
      const std::vector<ColumnFamilyOptions>& cf_opts,
      const std::string& file_name, Env* env);

-  static Status VerifyDBOptions(const DBOptions& base_opt,
-                                const DBOptions& new_opt);
+  static Status VerifyDBOptions(
+      const DBOptions& base_opt, const DBOptions& new_opt,
+      const std::unordered_map<std::string, std::string>* new_opt_map =
+          nullptr);

-  static Status VerifyCFOptions(const ColumnFamilyOptions& base_opt,
-                                const ColumnFamilyOptions& new_opt);
+  static Status VerifyCFOptions(
+      const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt,
+      const std::unordered_map<std::string, std::string>* new_opt_map =
+          nullptr);

  static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);

@ -97,8 +108,10 @@ class RocksDBOptionsParser {

 private:
  DBOptions db_opt_;
+  std::unordered_map<std::string, std::string> db_opt_map_;
  std::vector<std::string> cf_names_;
  std::vector<ColumnFamilyOptions> cf_opts_;
+  std::vector<std::unordered_map<std::string, std::string>> cf_opt_maps_;
  bool has_version_section_;
  bool has_db_options_;
  bool has_default_cf_options_;
--- a/util/options_test.cc
+++ b/util/options_test.cc
@ -16,7 +16,9 @@
 #include <inttypes.h>

 #include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
 #include "rocksdb/convenience.h"
+#include "rocksdb/merge_operator.h"
 #include "rocksdb/options.h"
 #include "rocksdb/table.h"
 #include "rocksdb/utilities/leveldb_options.h"
@ -748,7 +750,132 @@ TEST_F(OptionsTest, DBOptionsSerialization) {
 }

 namespace {
+CompressionType RandomCompressionType(Random* rnd) {
+  return static_cast<CompressionType>(rnd->Uniform(6));
+}
+
+void RandomCompressionTypeVector(const size_t count,
+                                 std::vector<CompressionType>* types,
+                                 Random* rnd) {
+  types->clear();
+  for (size_t i = 0; i < count; ++i) {
+    types->emplace_back(RandomCompressionType(rnd));
+  }
+}
+
+const SliceTransform* RandomSliceTransform(Random* rnd, int pre_defined = -1) {
+  int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(4);
+  switch (random_num) {
+    case 0:
+      return NewFixedPrefixTransform(rnd->Uniform(20) + 1);
+    case 1:
+      return NewCappedPrefixTransform(rnd->Uniform(20) + 1);
+    case 2:
+      return NewNoopTransform();
+    default:
+      return nullptr;
+  }
+}
+
+TableFactory* RandomTableFactory(Random* rnd, int pre_defined = -1) {
+  int random_num = pre_defined >= 0 ? pre_defined : rnd->Uniform(3);
+  switch (random_num) {
+    case 0:
+      return NewPlainTableFactory();
+    case 1:
+      return NewCuckooTableFactory();
+    default:
+      return NewBlockBasedTableFactory();
+  }
+}
+
+std::string RandomString(Random* rnd, const size_t len) {
+  std::stringstream ss;
+  for (size_t i = 0; i < len; ++i) {
+    ss << static_cast<char>(rnd->Uniform(26) + 'a');
+  }
+  return ss.str();
+}
+
+class ChanglingMergeOperator : public MergeOperator {
+ public:
+  explicit ChanglingMergeOperator(const std::string& name)
+      : name_(name + "MergeOperator") {}
+  ~ChanglingMergeOperator() {}
+
+  void SetName(const std::string& name) { name_ = name; }
+
+  virtual bool FullMerge(const Slice& key, const Slice* existing_value,
+                         const std::deque<std::string>& operand_list,
+                         std::string* new_value,
+                         Logger* logger) const override {
+    return false;
+  }
+  virtual bool PartialMergeMulti(const Slice& key,
+                                 const std::deque<Slice>& operand_list,
+                                 std::string* new_value,
+                                 Logger* logger) const override {
+    return false;
+  }
+  virtual const char* Name() const override { return name_.c_str(); }
+
+ protected:
+  std::string name_;
+};
+
+MergeOperator* RandomMergeOperator(Random* rnd) {
+  return new ChanglingMergeOperator(RandomString(rnd, 10));
+}
+
+class ChanglingCompactionFilter : public CompactionFilter {
+ public:
+  explicit ChanglingCompactionFilter(const std::string& name)
+      : name_(name + "CompactionFilter") {}
+  ~ChanglingCompactionFilter() {}
+
+  void SetName(const std::string& name) { name_ = name; }
+
+  bool Filter(int level, const Slice& key, const Slice& existing_value,
+              std::string* new_value, bool* value_changed) const override {
+    return false;
+  }
+
+  const char* Name() const override { return name_.c_str(); }
+
+ private:
+  std::string name_;
+};
+
+CompactionFilter* RandomCompactionFilter(Random* rnd) {
+  return new ChanglingCompactionFilter(RandomString(rnd, 10));
+}
+
+class ChanglingCompactionFilterFactory : public CompactionFilterFactory {
+ public:
+  explicit ChanglingCompactionFilterFactory(const std::string& name)
+      : name_(name + "CompactionFilterFactory") {}
+  ~ChanglingCompactionFilterFactory() {}
+
+  void SetName(const std::string& name) { name_ = name; }
+
+  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
+      const CompactionFilter::Context& context) override {
+    return std::unique_ptr<CompactionFilter>();
+  }

+  // Returns a name that identifies this compaction filter factory.
+  const char* Name() const override { return name_.c_str(); }
+
+ protected:
+  std::string name_;
+};
+
+CompactionFilterFactory* RandomCompactionFilterFactory(Random* rnd) {
+  return new ChanglingCompactionFilterFactory(RandomString(rnd, 10));
+}
+
+// Note that the caller is responsible for releasing non-null
+// cf_opt->compaction_filter.
 void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {
  cf_opt->compaction_style = (CompactionStyle)(rnd->Uniform(4));

@ -803,6 +930,21 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, Random* rnd) {

  // unsigned int options
  cf_opt->rate_limit_delay_max_milliseconds = rnd->Uniform(10000);
+
+  // pointer typed options
+  cf_opt->prefix_extractor.reset(RandomSliceTransform(rnd));
+  cf_opt->table_factory.reset(RandomTableFactory(rnd));
+  cf_opt->merge_operator.reset(RandomMergeOperator(rnd));
+  if (cf_opt->compaction_filter) {
+    delete cf_opt->compaction_filter;
+  }
+  cf_opt->compaction_filter = RandomCompactionFilter(rnd);
+  cf_opt->compaction_filter_factory.reset(RandomCompactionFilterFactory(rnd));
+
+  // custom typed options
+  cf_opt->compression = RandomCompressionType(rnd);
+  RandomCompressionTypeVector(cf_opt->num_levels,
+                              &cf_opt->compression_per_level, rnd);
 }

 }  // namespace
@ -824,6 +966,9 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
  ASSERT_OK(GetColumnFamilyOptionsFromString(
      ColumnFamilyOptions(), base_options_file_content, &new_opt));
  ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(base_opt, new_opt));
+  if (base_opt.compaction_filter) {
+    delete base_opt.compaction_filter;
+  }
 }

 #endif  // !ROCKSDB_LITE
@ -1268,12 +1413,101 @@ TEST_F(OptionsParserTest, ParseVersion) {
  }
 }

+void VerifyCFPointerTypedOptions(
+    ColumnFamilyOptions* base_cf_opt, const ColumnFamilyOptions* new_cf_opt,
+    const std::unordered_map<std::string, std::string>* new_cf_opt_map) {
+  std::string name_buffer;
+  ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+                                                  new_cf_opt_map));
+
+  // change the name of merge operator back-and-forth
+  {
+    auto* merge_operator = dynamic_cast<ChanglingMergeOperator*>(
+        base_cf_opt->merge_operator.get());
+    if (merge_operator != nullptr) {
+      name_buffer = merge_operator->Name();
+      // change the name  and expect non-ok status
+      merge_operator->SetName("some-other-name");
+      ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+          *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+      // change the name back and expect ok status
+      merge_operator->SetName(name_buffer);
+      ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+                                                      new_cf_opt_map));
+    }
+  }
+
+  // change the name of the compaction filter factory back-and-forth
+  {
+    auto* compaction_filter_factory =
+        dynamic_cast<ChanglingCompactionFilterFactory*>(
+            base_cf_opt->compaction_filter_factory.get());
+    if (compaction_filter_factory != nullptr) {
+      name_buffer = compaction_filter_factory->Name();
+      // change the name and expect non-ok status
+      compaction_filter_factory->SetName("some-other-name");
+      ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+          *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+      // change the name back and expect ok status
+      compaction_filter_factory->SetName(name_buffer);
+      ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+                                                      new_cf_opt_map));
+    }
+  }
+
+  // test by setting compaction_filter to nullptr
+  {
+    auto* tmp_compaction_filter = base_cf_opt->compaction_filter;
+    if (tmp_compaction_filter != nullptr) {
+      base_cf_opt->compaction_filter = nullptr;
+      // set compaction_filter to nullptr and expect non-ok status
+      ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+          *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+      // set the value back and expect ok status
+      base_cf_opt->compaction_filter = tmp_compaction_filter;
+      ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+                                                      new_cf_opt_map));
+    }
+  }
+
+  // test by setting table_factory to nullptr
+  {
+    auto tmp_table_factory = base_cf_opt->table_factory;
+    if (tmp_table_factory != nullptr) {
+      base_cf_opt->table_factory.reset();
+      // set table_factory to nullptr and expect non-ok status
+      ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+          *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+      // set the value back and expect ok status
+      base_cf_opt->table_factory = tmp_table_factory;
+      ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+                                                      new_cf_opt_map));
+    }
+  }
+
+  // test by setting memtable_factory to nullptr
+  {
+    auto tmp_memtable_factory = base_cf_opt->memtable_factory;
+    if (tmp_memtable_factory != nullptr) {
+      base_cf_opt->memtable_factory.reset();
+      // set memtable_factory to nullptr and expect non-ok status
+      ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
+          *base_cf_opt, *new_cf_opt, new_cf_opt_map));
+      // set the value back and expect ok status
+      base_cf_opt->memtable_factory = tmp_memtable_factory;
+      ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*base_cf_opt, *new_cf_opt,
+                                                      new_cf_opt_map));
+    }
+  }
+}
+
 TEST_F(OptionsParserTest, DumpAndParse) {
  DBOptions base_db_opt;
  std::vector<ColumnFamilyOptions> base_cf_opts;
-  std::vector<std::string> cf_names = {
-      // special characters are also included.
-      "default", "p\\i\\k\\a\\chu\\\\\\", "###rocksdb#1-testcf#2###"};
+  std::vector<std::string> cf_names = {"default", "cf1", "cf2", "cf3",
+                                       "c:f:4:4:4"
+                                       "p\\i\\k\\a\\chu\\\\\\",
+                                       "###rocksdb#1-testcf#2###"};
  const int num_cf = static_cast<int>(cf_names.size());
  Random rnd(302);
  RandomInitDBOptions(&base_db_opt, &rnd);
@ -1282,6 +1516,12 @@ TEST_F(OptionsParserTest, DumpAndParse) {
    ColumnFamilyOptions cf_opt;
    Random cf_rnd(0xFB + c);
    RandomInitCFOptions(&cf_opt, &cf_rnd);
+    if (c < 4) {
+      cf_opt.prefix_extractor.reset(RandomSliceTransform(&rnd, c));
+    }
+    if (c < 3) {
+      cf_opt.table_factory.reset(RandomTableFactory(&rnd, c));
+    }
    base_cf_opts.emplace_back(cf_opt);
  }

@ -1300,13 +1540,29 @@ TEST_F(OptionsParserTest, DumpAndParse) {
  for (int c = 0; c < num_cf; ++c) {
    const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
    ASSERT_NE(cf_opt, nullptr);
-    ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(*cf_opt, base_cf_opts[c]));
+    ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
+        base_cf_opts[c], *cf_opt, &(parser.cf_opt_maps()->at(c))));
+  }
+
+  // Further verify pointer-typed options
+  for (int c = 0; c < num_cf; ++c) {
+    const auto* cf_opt = parser.GetCFOptions(cf_names[c]);
+    ASSERT_NE(cf_opt, nullptr);
+    VerifyCFPointerTypedOptions(&base_cf_opts[c], cf_opt,
+                                &(parser.cf_opt_maps()->at(c)));
  }
+
  ASSERT_EQ(parser.GetCFOptions("does not exist"), nullptr);

  base_db_opt.max_open_files++;
  ASSERT_NOK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
      base_db_opt, cf_names, base_cf_opts, kOptionsFileName, env_.get()));
+
+  for (int c = 0; c < num_cf; ++c) {
+    if (base_cf_opts[c].compaction_filter) {
+      delete base_cf_opts[c].compaction_filter;
+    }
+  }
 }

 namespace {
--- a/util/slice.cc
+++ b/util/slice.cc
@ -25,6 +25,11 @@ class FixedPrefixTransform : public SliceTransform {
 public:
  explicit FixedPrefixTransform(size_t prefix_len)
      : prefix_len_(prefix_len),
+        // Note that if any part of the name format changes, it will require
+        // changes on options_helper in order to make RocksDBOptionsParser work
+        // for the new change.
+        // TODO(yhchiang): move serialization / deserializaion code inside
+        // the class implementation itself.
        name_("rocksdb.FixedPrefix." + ToString(prefix_len_)) {}

  virtual const char* Name() const override { return name_.c_str(); }
@ -55,6 +60,11 @@ class CappedPrefixTransform : public SliceTransform {
 public:
  explicit CappedPrefixTransform(size_t cap_len)
      : cap_len_(cap_len),
+        // Note that if any part of the name format changes, it will require
+        // changes on options_helper in order to make RocksDBOptionsParser work
+        // for the new change.
+        // TODO(yhchiang): move serialization / deserializaion code inside
+        // the class implementation itself.
        name_("rocksdb.CappedPrefix." + ToString(cap_len_)) {}

  virtual const char* Name() const override { return name_.c_str(); }