Merge pull request #797 from SherlockNoMad/optionHelper

Support PlainTableOption in option_helper.cc

include/rocksdb/convenience.h

@@ -43,6 +43,12 @@ Status GetBlockBasedTableOptionsFromMap(
     BlockBasedTableOptions* new_table_options,
     bool input_strings_escaped = false);
 
+Status GetPlainTableOptionsFromMap(
+    const PlainTableOptions& table_options,
+    const std::unordered_map<std::string, std::string>& opts_map,
+    PlainTableOptions* new_table_options,
+    bool input_strings_escaped = false);
+
 // Take a string representation of option names and  values, apply them into the
 // base_options, and return the new options as a result. The string has the
 // following format:
@@ -74,6 +80,11 @@ Status GetBlockBasedTableOptionsFromString(
     const std::string& opts_str,
     BlockBasedTableOptions* new_table_options);
 
+Status GetPlainTableOptionsFromString(
+    const PlainTableOptions& table_options,
+    const std::string& opts_str,
+    PlainTableOptions* new_table_options);
+
 Status GetOptionsFromString(const Options& base_options,
                             const std::string& opts_str, Options* new_options);
 

table/plain_table_factory.cc

@@ -21,8 +21,9 @@ Status PlainTableFactory::NewTableReader(
   return PlainTableReader::Open(
       table_reader_options.ioptions, table_reader_options.env_options,
       table_reader_options.internal_comparator, std::move(file), file_size,
-      table, bloom_bits_per_key_, hash_table_ratio_, index_sparseness_,
-      huge_page_tlb_size_, full_scan_mode_);
+      table, table_options_.bloom_bits_per_key, table_options_.hash_table_ratio, 
+      table_options_.index_sparseness, table_options_.huge_page_tlb_size, 
+      table_options_.full_scan_mode);
 }
 
 TableBuilder* PlainTableFactory::NewTableBuilder(
@@ -35,9 +36,10 @@ TableBuilder* PlainTableFactory::NewTableBuilder(
   return new PlainTableBuilder(
       table_builder_options.ioptions,
       table_builder_options.int_tbl_prop_collector_factories, column_family_id,
-      file, user_key_len_, encoding_type_, index_sparseness_,
-      bloom_bits_per_key_, 6, huge_page_tlb_size_, hash_table_ratio_,
-      store_index_in_file_);
+      file, table_options_.user_key_len, table_options_.encoding_type, 
+      table_options_.index_sparseness, table_options_.bloom_bits_per_key, 6, 
+      table_options_.huge_page_tlb_size, table_options_.hash_table_ratio,
+      table_options_.store_index_in_file);
 }
 
 std::string PlainTableFactory::GetPrintableTableOptions() const {
@@ -47,32 +49,36 @@ std::string PlainTableFactory::GetPrintableTableOptions() const {
   char buffer[kBufferSize];
 
   snprintf(buffer, kBufferSize, "  user_key_len: %u\n",
-           user_key_len_);
+           table_options_.user_key_len);
   ret.append(buffer);
   snprintf(buffer, kBufferSize, "  bloom_bits_per_key: %d\n",
-           bloom_bits_per_key_);
+           table_options_.bloom_bits_per_key);
   ret.append(buffer);
   snprintf(buffer, kBufferSize, "  hash_table_ratio: %lf\n",
-           hash_table_ratio_);
+           table_options_.hash_table_ratio);
   ret.append(buffer);
   snprintf(buffer, kBufferSize, "  index_sparseness: %" ROCKSDB_PRIszt "\n",
-           index_sparseness_);
+           table_options_.index_sparseness);
   ret.append(buffer);
   snprintf(buffer, kBufferSize, "  huge_page_tlb_size: %" ROCKSDB_PRIszt "\n",
-           huge_page_tlb_size_);
+           table_options_.huge_page_tlb_size);
   ret.append(buffer);
   snprintf(buffer, kBufferSize, "  encoding_type: %d\n",
-           encoding_type_);
+           table_options_.encoding_type);
   ret.append(buffer);
   snprintf(buffer, kBufferSize, "  full_scan_mode: %d\n",
-           full_scan_mode_);
+           table_options_.full_scan_mode);
   ret.append(buffer);
   snprintf(buffer, kBufferSize, "  store_index_in_file: %d\n",
-           store_index_in_file_);
+           table_options_.store_index_in_file);
   ret.append(buffer);
   return ret;
 }
 
+const PlainTableOptions& PlainTableFactory::GetTableOptions() const {
+  return table_options_;
+}
+
 extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) {
   return new PlainTableFactory(options);
 }

table/plain_table_factory.h

@@ -142,27 +142,24 @@ class PlainTableFactory : public TableFactory {
   // huge_page_tlb_size determines whether to allocate hash indexes from huge
   // page TLB and the page size if allocating from there. See comments of
   // Arena::AllocateAligned() for details.
-  explicit PlainTableFactory(const PlainTableOptions& options =
-                                 PlainTableOptions())
-      : user_key_len_(options.user_key_len),
-        bloom_bits_per_key_(options.bloom_bits_per_key),
-        hash_table_ratio_(options.hash_table_ratio),
-        index_sparseness_(options.index_sparseness),
-        huge_page_tlb_size_(options.huge_page_tlb_size),
-        encoding_type_(options.encoding_type),
-        full_scan_mode_(options.full_scan_mode),
-        store_index_in_file_(options.store_index_in_file) {}
+  explicit PlainTableFactory(const PlainTableOptions& table_options =
+                             PlainTableOptions())
+      : table_options_(table_options) {};
+
   const char* Name() const override { return "PlainTable"; }
   Status NewTableReader(const TableReaderOptions& table_reader_options,
                         unique_ptr<RandomAccessFileReader>&& file,
                         uint64_t file_size,
                         unique_ptr<TableReader>* table) const override;
+  
   TableBuilder* NewTableBuilder(
       const TableBuilderOptions& table_builder_options,
       uint32_t column_family_id, WritableFileWriter* file) const override;
 
   std::string GetPrintableTableOptions() const override;
 
+  const PlainTableOptions& GetTableOptions() const;
+
   static const char kValueTypeSeqId0 = 0xFF;
 
   // Sanitizes the specified DB Options.
@@ -172,14 +169,7 @@ class PlainTableFactory : public TableFactory {
   }
 
  private:
-  uint32_t user_key_len_;
-  int bloom_bits_per_key_;
-  double hash_table_ratio_;
-  size_t index_sparseness_;
-  size_t huge_page_tlb_size_;
-  EncodingType encoding_type_;
-  bool full_scan_mode_;
-  bool store_index_in_file_;
+  PlainTableOptions table_options_;
 };
 
 }  // namespace rocksdb

util/options_helper.cc

@@ -20,6 +20,7 @@
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/table.h"
 #include "table/block_based_table_factory.h"
+#include "table/plain_table_factory.h"
 #include "util/logging.h"
 #include "util/string_util.h"
 
@@ -200,6 +201,31 @@ bool ParseBlockBasedTableIndexType(const std::string& type,
   return true;
 }
 
+bool SerializeEncodingType(
+  const EncodingType& type, std::string* value) {
+  switch (type) {
+    case EncodingType::kPlain:
+      *value = "kPlain";
+      return true;
+    case EncodingType::kPrefix:
+      *value = "kPrefix";
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool PraseEncodingType(const std::string& type, EncodingType* value) {
+  if (type == "kPlain") {
+    *value = EncodingType::kPlain;
+  } else if (type == "kPrefix") {
+    *value = EncodingType::kPrefix;
+  } else {
+    return false;
+  }
+  return true;
+}
+
 static std::unordered_map<std::string, ChecksumType> checksum_type_map = {
     {"kNoChecksum", kNoChecksum}, {"kCRC32c", kCRC32c}, {"kxxHash", kxxHash}};
 
@@ -448,6 +474,10 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
       return ParseBlockBasedTableIndexType(
           value,
           reinterpret_cast<BlockBasedTableOptions::IndexType*>(opt_address));
+    case OptionType::kEncodingType:
+      return PraseEncodingType(
+          value,
+          reinterpret_cast<EncodingType*>(opt_address));
     default:
       return false;
   }
@@ -569,6 +599,9 @@ bool SerializeSingleOptionHelper(const char* opt_address,
       *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
       break;
     }
+    case OptionType::kEncodingType:
+      return SerializeEncodingType(
+          *reinterpret_cast<const EncodingType*>(opt_address), value);
     default:
       return false;
   }
@@ -774,9 +807,9 @@ Status StringToMap(const std::string& opts_str,
 bool ParseColumnFamilyOption(const std::string& name,
                              const std::string& org_value,
                              ColumnFamilyOptions* new_options,
-                             bool input_string_escaped = false) {
+                             bool input_strings_escaped = false) {
   const std::string& value =
-      input_string_escaped ? UnescapeOptionString(org_value) : org_value;
+      input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
   try {
     if (name == "max_bytes_for_level_multiplier_additional") {
       new_options->max_bytes_for_level_multiplier_additional.clear();
@@ -807,6 +840,20 @@ bool ParseColumnFamilyOption(const std::string& name,
         return false;
       }
       new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt));
+    } else if (name == "plain_table_factory") {
+      // Nested options
+      PlainTableOptions table_opt, base_table_options;
+      auto plain_table_factory = dynamic_cast<PlainTableFactory*>(
+          new_options->table_factory.get());
+      if (plain_table_factory != nullptr) {
+        base_table_options = plain_table_factory->GetTableOptions();
+      }
+      Status table_opt_s = GetPlainTableOptionsFromString(
+          base_table_options, value, &table_opt);
+      if (!table_opt_s.ok()) {
+        return false;
+      }
+      new_options->table_factory.reset(NewPlainTableFactory(table_opt));
     } else if (name == "compression_opts") {
       size_t start = 0;
       size_t end = value.find(':');
@@ -991,9 +1038,9 @@ Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf,
 }
 
 bool ParseDBOption(const std::string& name, const std::string& org_value,
-                   DBOptions* new_options, bool input_string_escaped = false) {
+                   DBOptions* new_options, bool input_strings_escaped = false) {
   const std::string& value =
-      input_string_escaped ? UnescapeOptionString(org_value) : org_value;
+      input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
   try {
     if (name == "rate_limiter_bytes_per_sec") {
       new_options->rate_limiter.reset(
@@ -1020,10 +1067,10 @@ bool ParseDBOption(const std::string& name, const std::string& org_value,
 std::string ParseBlockBasedTableOption(const std::string& name,
                                        const std::string& org_value,
                                        BlockBasedTableOptions* new_options,
-                                       bool input_string_escaped = false) {
+                                       bool input_strings_escaped = false) {
   const std::string& value =
-      input_string_escaped ? UnescapeOptionString(org_value) : org_value;
-  if (!input_string_escaped) {
+      input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
+  if (!input_strings_escaped) {
     // if the input string is not escaped, it means this function is
     // invoked from SetOptions, which takes the old format.
     if (name == "block_cache") {
@@ -1064,6 +1111,24 @@ std::string ParseBlockBasedTableOption(const std::string& name,
   return "";
 }
 
+std::string ParsePlainTableOptions(const std::string& name,
+                                   const std::string& org_value,
+                                   PlainTableOptions* new_option,
+                                   bool input_strings_escaped = false) {
+  const std::string& value = 
+      input_strings_escaped ? UnescapeOptionString(org_value) : org_value;
+  const auto iter = plain_table_type_info.find(name);
+  if (iter == plain_table_type_info.end()) {
+    return "Unrecognized option";
+  }
+  const auto& opt_info = iter->second;
+  if (!ParseOptionHelper(reinterpret_cast<char*>(new_option) + opt_info.offset,
+                         opt_info.type, value)) {
+    return "Invalid value";
+  }
+  return "";
+}
+
 Status GetBlockBasedTableOptionsFromMap(
     const BlockBasedTableOptions& table_options,
     const std::unordered_map<std::string, std::string>& opts_map,
@@ -1105,46 +1170,41 @@ Status GetBlockBasedTableOptionsFromString(
 Status GetPlainTableOptionsFromMap(
     const PlainTableOptions& table_options,
     const std::unordered_map<std::string, std::string>& opts_map,
-    PlainTableOptions* new_table_options) {
+    PlainTableOptions* new_table_options, bool input_strings_escaped) {
   assert(new_table_options);
   *new_table_options = table_options;
-
   for (const auto& o : opts_map) {
-    try {
-      if (o.first == "user_key_len") {
-        new_table_options->user_key_len = ParseUint32(o.second);
-      } else if (o.first == "bloom_bits_per_key") {
-        new_table_options->bloom_bits_per_key = ParseInt(o.second);
-      } else if (o.first == "hash_table_ratio") {
-        new_table_options->hash_table_ratio = ParseDouble(o.second);
-      } else if (o.first == "index_sparseness") {
-        new_table_options->index_sparseness = ParseSizeT(o.second);
-      } else if (o.first == "huge_page_tlb_size") {
-        new_table_options->huge_page_tlb_size = ParseSizeT(o.second);
-      } else if (o.first == "encoding_type") {
-        if (o.second == "kPlain") {
-          new_table_options->encoding_type = kPlain;
-        } else if (o.second == "kPrefix") {
-          new_table_options->encoding_type = kPrefix;
-        } else {
-          throw std::invalid_argument("Unknown encoding_type: " + o.second);
-        }
-      } else if (o.first == "full_scan_mode") {
-        new_table_options->full_scan_mode = ParseBoolean(o.first, o.second);
-      } else if (o.first == "store_index_in_file") {
-        new_table_options->store_index_in_file =
-            ParseBoolean(o.first, o.second);
-      } else {
-        return Status::InvalidArgument("Unrecognized option: " + o.first);
+    auto error_message = ParsePlainTableOptions(
+        o.first, o.second, new_table_options, input_strings_escaped);
+    if (error_message != "") {
+      const auto iter = plain_table_type_info.find(o.first);
+      if (iter == plain_table_type_info.end() ||
+          !input_strings_escaped ||// !input_strings_escaped indicates
+                                   // the old API, where everything is
+                                   // parsable.
+          (iter->second.verification != OptionVerificationType::kByName &&
+           iter->second.verification != OptionVerificationType::kDeprecated)) {
+        return Status::InvalidArgument("Can't parse PlainTableOptions:",
+                                        o.first + " " + error_message);
       }
-    } catch (std::exception& e) {
-      return Status::InvalidArgument("error parsing " + o.first + ":" +
-                                     std::string(e.what()));
     }
   }
   return Status::OK();
 }
 
+Status GetPlainTableOptionsFromString(
+    const PlainTableOptions& table_options,
+    const std::string& opts_str,
+    PlainTableOptions* new_table_options) {
+  std::unordered_map<std::string, std::string> opts_map;
+  Status s = StringToMap(opts_str, &opts_map);
+  if (!s.ok()) {
+    return s;
+  }
+  return GetPlainTableOptionsFromMap(table_options, opts_map, 
+                                     new_table_options);
+}
+
 Status GetColumnFamilyOptionsFromMap(
     const ColumnFamilyOptions& base_options,
     const std::unordered_map<std::string, std::string>& opts_map,
@@ -1239,6 +1299,15 @@ Status GetTableFactoryFromMap(
     }
     table_factory->reset(new BlockBasedTableFactory(bbt_opt));
     return Status::OK();
+  } else if (factory_name == PlainTableFactory().Name()) {
+    PlainTableOptions pt_opt;
+    s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, 
+                                    &pt_opt, true);
+    if (!s.ok()) {
+      return s;
+    }
+    table_factory->reset(new PlainTableFactory(pt_opt));
+    return Status::OK();
   }
   // Return OK for not supported table factories as TableFactory
   // Deserialization is optional.

util/options_helper.h

@@ -87,6 +87,7 @@ enum class OptionType {
   kFilterPolicy,
   kFlushBlockPolicyFactory,
   kChecksumType,
+  kEncodingType,
   kUnknown
 };
 
@@ -462,6 +463,34 @@ static std::unordered_map<std::string,
     {"format_version",
      {offsetof(struct BlockBasedTableOptions, format_version),
       OptionType::kUInt32T, OptionVerificationType::kNormal}}};
+
+static std::unordered_map<std::string, 
+                          OptionTypeInfo> plain_table_type_info = {
+      {"user_key_len", 
+       {offsetof(struct PlainTableOptions, user_key_len),
+        OptionType::kUInt32T, OptionVerificationType::kNormal}},
+      {"bloom_bits_per_key", 
+       {offsetof(struct PlainTableOptions, bloom_bits_per_key),
+        OptionType::kInt, OptionVerificationType::kNormal}},
+      {"hash_table_ratio", 
+       {offsetof(struct PlainTableOptions, hash_table_ratio),
+        OptionType::kDouble, OptionVerificationType::kNormal}},
+      {"index_sparseness", 
+       {offsetof(struct PlainTableOptions, index_sparseness),
+        OptionType::kSizeT, OptionVerificationType::kNormal}},
+      {"huge_page_tlb_size", 
+       {offsetof(struct PlainTableOptions, huge_page_tlb_size),
+        OptionType::kSizeT, OptionVerificationType::kNormal}},
+      {"encoding_type", 
+       {offsetof(struct PlainTableOptions, encoding_type),
+        OptionType::kEncodingType, OptionVerificationType::kByName}},
+      {"full_scan_mode", 
+       {offsetof(struct PlainTableOptions, full_scan_mode),
+        OptionType::kBoolean, OptionVerificationType::kNormal}},
+      {"store_index_in_file", 
+       {offsetof(struct PlainTableOptions, store_index_in_file),
+        OptionType::kBoolean, OptionVerificationType::kNormal}}};  
+
 }  // namespace rocksdb
 
 #endif  // !ROCKSDB_LITE

util/options_test.cc

@@ -23,6 +23,7 @@
 #include "rocksdb/table.h"
 #include "rocksdb/utilities/leveldb_options.h"
 #include "table/block_based_table_factory.h"
+#include "table/plain_table_factory.h"
 #include "util/options_helper.h"
 #include "util/options_parser.h"
 #include "util/random.h"
@@ -590,6 +591,23 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
   ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt,
               "optimize_filters_for_hits=junk",
               &new_cf_opt));
+
+  // Nested plain table options
+  // Emtpy
+  ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+            "write_buffer_size=10;max_write_buffer_number=16;"
+            "plain_table_factory={};arena_block_size=1024",
+            &new_cf_opt));
+  ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+  ASSERT_EQ(new_cf_opt.table_factory->Name(), "PlainTable");
+  // Non-empty
+  ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt,
+            "write_buffer_size=10;max_write_buffer_number=16;"
+            "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};"
+            "arena_block_size=1024",
+            &new_cf_opt));
+  ASSERT_TRUE(new_cf_opt.table_factory != nullptr);
+  ASSERT_EQ(new_cf_opt.table_factory->Name(), "PlainTable");
 }
 #endif  // !ROCKSDB_LITE
 
@@ -648,6 +666,40 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
 }
 #endif  // !ROCKSDB_LITE
 
+
+#ifndef ROCKSDB_LITE  // GetPlainTableOptionsFromString is not supported
+TEST_F(OptionsTest, GetPlainTableOptionsFromString) {
+  PlainTableOptions table_opt;
+  PlainTableOptions new_opt;
+  // make sure default values are overwritten by something else
+  ASSERT_OK(GetPlainTableOptionsFromString(table_opt,
+            "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+            "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;"
+            "full_scan_mode=true;store_index_in_file=true",
+            &new_opt));
+  ASSERT_EQ(new_opt.user_key_len, 66);
+  ASSERT_EQ(new_opt.bloom_bits_per_key, 20);
+  ASSERT_EQ(new_opt.hash_table_ratio, 0.5);
+  ASSERT_EQ(new_opt.index_sparseness, 8);
+  ASSERT_EQ(new_opt.huge_page_tlb_size, 4);
+  ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix);
+  ASSERT_TRUE(new_opt.full_scan_mode);
+  ASSERT_TRUE(new_opt.store_index_in_file);
+
+  // unknown option
+  ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
+             "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+             "bad_option=1",
+             &new_opt));
+
+  // unrecognized EncodingType
+  ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
+             "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
+             "encoding_type=kPrefixXX",
+             &new_opt));  
+}
+#endif  // !ROCKSDB_LITE
+
 #ifndef ROCKSDB_LITE  // GetOptionsFromString is not supported in RocksDB Lite
 TEST_F(OptionsTest, GetOptionsFromStringTest) {
   Options base_options, new_options;