diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h index f7b845555..15df53204 100644 --- a/include/rocksdb/convenience.h +++ b/include/rocksdb/convenience.h @@ -43,6 +43,12 @@ Status GetBlockBasedTableOptionsFromMap( BlockBasedTableOptions* new_table_options, bool input_strings_escaped = false); +Status GetPlainTableOptionsFromMap( + const PlainTableOptions& table_options, + const std::unordered_map& opts_map, + PlainTableOptions* new_table_options, + bool input_strings_escaped = false); + // Take a string representation of option names and values, apply them into the // base_options, and return the new options as a result. The string has the // following format: @@ -74,6 +80,11 @@ Status GetBlockBasedTableOptionsFromString( const std::string& opts_str, BlockBasedTableOptions* new_table_options); +Status GetPlainTableOptionsFromString( + const PlainTableOptions& table_options, + const std::string& opts_str, + PlainTableOptions* new_table_options); + Status GetOptionsFromString(const Options& base_options, const std::string& opts_str, Options* new_options); diff --git a/table/plain_table_factory.cc b/table/plain_table_factory.cc index 4d25f2150..f5f3a7692 100644 --- a/table/plain_table_factory.cc +++ b/table/plain_table_factory.cc @@ -21,8 +21,9 @@ Status PlainTableFactory::NewTableReader( return PlainTableReader::Open( table_reader_options.ioptions, table_reader_options.env_options, table_reader_options.internal_comparator, std::move(file), file_size, - table, bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, - huge_page_tlb_size_, full_scan_mode_); + table, table_options_.bloom_bits_per_key, table_options_.hash_table_ratio, + table_options_.index_sparseness, table_options_.huge_page_tlb_size, + table_options_.full_scan_mode); } TableBuilder* PlainTableFactory::NewTableBuilder( @@ -35,9 +36,10 @@ TableBuilder* PlainTableFactory::NewTableBuilder( return new PlainTableBuilder( table_builder_options.ioptions, table_builder_options.int_tbl_prop_collector_factories, column_family_id, - file, user_key_len_, encoding_type_, index_sparseness_, - bloom_bits_per_key_, 6, huge_page_tlb_size_, hash_table_ratio_, - store_index_in_file_); + file, table_options_.user_key_len, table_options_.encoding_type, + table_options_.index_sparseness, table_options_.bloom_bits_per_key, 6, + table_options_.huge_page_tlb_size, table_options_.hash_table_ratio, + table_options_.store_index_in_file); } std::string PlainTableFactory::GetPrintableTableOptions() const { @@ -47,32 +49,36 @@ std::string PlainTableFactory::GetPrintableTableOptions() const { char buffer[kBufferSize]; snprintf(buffer, kBufferSize, " user_key_len: %u\n", - user_key_len_); + table_options_.user_key_len); ret.append(buffer); snprintf(buffer, kBufferSize, " bloom_bits_per_key: %d\n", - bloom_bits_per_key_); + table_options_.bloom_bits_per_key); ret.append(buffer); snprintf(buffer, kBufferSize, " hash_table_ratio: %lf\n", - hash_table_ratio_); + table_options_.hash_table_ratio); ret.append(buffer); snprintf(buffer, kBufferSize, " index_sparseness: %" ROCKSDB_PRIszt "\n", - index_sparseness_); + table_options_.index_sparseness); ret.append(buffer); snprintf(buffer, kBufferSize, " huge_page_tlb_size: %" ROCKSDB_PRIszt "\n", - huge_page_tlb_size_); + table_options_.huge_page_tlb_size); ret.append(buffer); snprintf(buffer, kBufferSize, " encoding_type: %d\n", - encoding_type_); + table_options_.encoding_type); ret.append(buffer); snprintf(buffer, kBufferSize, " full_scan_mode: %d\n", - full_scan_mode_); + table_options_.full_scan_mode); ret.append(buffer); snprintf(buffer, kBufferSize, " store_index_in_file: %d\n", - store_index_in_file_); + table_options_.store_index_in_file); ret.append(buffer); return ret; } +const PlainTableOptions& PlainTableFactory::GetTableOptions() const { + return table_options_; +} + extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) { return new PlainTableFactory(options); } diff --git a/table/plain_table_factory.h b/table/plain_table_factory.h index 5d1125bb0..d21c9fd1e 100644 --- a/table/plain_table_factory.h +++ b/table/plain_table_factory.h @@ -142,27 +142,24 @@ class PlainTableFactory : public TableFactory { // huge_page_tlb_size determines whether to allocate hash indexes from huge // page TLB and the page size if allocating from there. See comments of // Arena::AllocateAligned() for details. - explicit PlainTableFactory(const PlainTableOptions& options = - PlainTableOptions()) - : user_key_len_(options.user_key_len), - bloom_bits_per_key_(options.bloom_bits_per_key), - hash_table_ratio_(options.hash_table_ratio), - index_sparseness_(options.index_sparseness), - huge_page_tlb_size_(options.huge_page_tlb_size), - encoding_type_(options.encoding_type), - full_scan_mode_(options.full_scan_mode), - store_index_in_file_(options.store_index_in_file) {} + explicit PlainTableFactory(const PlainTableOptions& table_options = + PlainTableOptions()) + : table_options_(table_options) {}; + const char* Name() const override { return "PlainTable"; } Status NewTableReader(const TableReaderOptions& table_reader_options, unique_ptr&& file, uint64_t file_size, unique_ptr* table) const override; + TableBuilder* NewTableBuilder( const TableBuilderOptions& table_builder_options, uint32_t column_family_id, WritableFileWriter* file) const override; std::string GetPrintableTableOptions() const override; + const PlainTableOptions& GetTableOptions() const; + static const char kValueTypeSeqId0 = 0xFF; // Sanitizes the specified DB Options. @@ -172,14 +169,7 @@ class PlainTableFactory : public TableFactory { } private: - uint32_t user_key_len_; - int bloom_bits_per_key_; - double hash_table_ratio_; - size_t index_sparseness_; - size_t huge_page_tlb_size_; - EncodingType encoding_type_; - bool full_scan_mode_; - bool store_index_in_file_; + PlainTableOptions table_options_; }; } // namespace rocksdb diff --git a/util/options_helper.cc b/util/options_helper.cc index b2e8dfe51..f1f04c481 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -20,6 +20,7 @@ #include "rocksdb/slice_transform.h" #include "rocksdb/table.h" #include "table/block_based_table_factory.h" +#include "table/plain_table_factory.h" #include "util/logging.h" #include "util/string_util.h" @@ -200,6 +201,31 @@ bool ParseBlockBasedTableIndexType(const std::string& type, return true; } +bool SerializeEncodingType( + const EncodingType& type, std::string* value) { + switch (type) { + case EncodingType::kPlain: + *value = "kPlain"; + return true; + case EncodingType::kPrefix: + *value = "kPrefix"; + return true; + default: + return false; + } +} + +bool PraseEncodingType(const std::string& type, EncodingType* value) { + if (type == "kPlain") { + *value = EncodingType::kPlain; + } else if (type == "kPrefix") { + *value = EncodingType::kPrefix; + } else { + return false; + } + return true; +} + static std::unordered_map checksum_type_map = { {"kNoChecksum", kNoChecksum}, {"kCRC32c", kCRC32c}, {"kxxHash", kxxHash}}; @@ -448,6 +474,10 @@ bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, return ParseBlockBasedTableIndexType( value, reinterpret_cast(opt_address)); + case OptionType::kEncodingType: + return PraseEncodingType( + value, + reinterpret_cast(opt_address)); default: return false; } @@ -569,6 +599,9 @@ bool SerializeSingleOptionHelper(const char* opt_address, *value = ptr->get() ? ptr->get()->Name() : kNullptrString; break; } + case OptionType::kEncodingType: + return SerializeEncodingType( + *reinterpret_cast(opt_address), value); default: return false; } @@ -774,9 +807,9 @@ Status StringToMap(const std::string& opts_str, bool ParseColumnFamilyOption(const std::string& name, const std::string& org_value, ColumnFamilyOptions* new_options, - bool input_string_escaped = false) { + bool input_strings_escaped = false) { const std::string& value = - input_string_escaped ? UnescapeOptionString(org_value) : org_value; + input_strings_escaped ? UnescapeOptionString(org_value) : org_value; try { if (name == "max_bytes_for_level_multiplier_additional") { new_options->max_bytes_for_level_multiplier_additional.clear(); @@ -807,6 +840,20 @@ bool ParseColumnFamilyOption(const std::string& name, return false; } new_options->table_factory.reset(NewBlockBasedTableFactory(table_opt)); + } else if (name == "plain_table_factory") { + // Nested options + PlainTableOptions table_opt, base_table_options; + auto plain_table_factory = dynamic_cast( + new_options->table_factory.get()); + if (plain_table_factory != nullptr) { + base_table_options = plain_table_factory->GetTableOptions(); + } + Status table_opt_s = GetPlainTableOptionsFromString( + base_table_options, value, &table_opt); + if (!table_opt_s.ok()) { + return false; + } + new_options->table_factory.reset(NewPlainTableFactory(table_opt)); } else if (name == "compression_opts") { size_t start = 0; size_t end = value.find(':'); @@ -991,9 +1038,9 @@ Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf, } bool ParseDBOption(const std::string& name, const std::string& org_value, - DBOptions* new_options, bool input_string_escaped = false) { + DBOptions* new_options, bool input_strings_escaped = false) { const std::string& value = - input_string_escaped ? UnescapeOptionString(org_value) : org_value; + input_strings_escaped ? UnescapeOptionString(org_value) : org_value; try { if (name == "rate_limiter_bytes_per_sec") { new_options->rate_limiter.reset( @@ -1020,10 +1067,10 @@ bool ParseDBOption(const std::string& name, const std::string& org_value, std::string ParseBlockBasedTableOption(const std::string& name, const std::string& org_value, BlockBasedTableOptions* new_options, - bool input_string_escaped = false) { + bool input_strings_escaped = false) { const std::string& value = - input_string_escaped ? UnescapeOptionString(org_value) : org_value; - if (!input_string_escaped) { + input_strings_escaped ? UnescapeOptionString(org_value) : org_value; + if (!input_strings_escaped) { // if the input string is not escaped, it means this function is // invoked from SetOptions, which takes the old format. if (name == "block_cache") { @@ -1064,6 +1111,24 @@ std::string ParseBlockBasedTableOption(const std::string& name, return ""; } +std::string ParsePlainTableOptions(const std::string& name, + const std::string& org_value, + PlainTableOptions* new_option, + bool input_strings_escaped = false) { + const std::string& value = + input_strings_escaped ? UnescapeOptionString(org_value) : org_value; + const auto iter = plain_table_type_info.find(name); + if (iter == plain_table_type_info.end()) { + return "Unrecognized option"; + } + const auto& opt_info = iter->second; + if (!ParseOptionHelper(reinterpret_cast(new_option) + opt_info.offset, + opt_info.type, value)) { + return "Invalid value"; + } + return ""; +} + Status GetBlockBasedTableOptionsFromMap( const BlockBasedTableOptions& table_options, const std::unordered_map& opts_map, @@ -1105,46 +1170,41 @@ Status GetBlockBasedTableOptionsFromString( Status GetPlainTableOptionsFromMap( const PlainTableOptions& table_options, const std::unordered_map& opts_map, - PlainTableOptions* new_table_options) { + PlainTableOptions* new_table_options, bool input_strings_escaped) { assert(new_table_options); *new_table_options = table_options; - for (const auto& o : opts_map) { - try { - if (o.first == "user_key_len") { - new_table_options->user_key_len = ParseUint32(o.second); - } else if (o.first == "bloom_bits_per_key") { - new_table_options->bloom_bits_per_key = ParseInt(o.second); - } else if (o.first == "hash_table_ratio") { - new_table_options->hash_table_ratio = ParseDouble(o.second); - } else if (o.first == "index_sparseness") { - new_table_options->index_sparseness = ParseSizeT(o.second); - } else if (o.first == "huge_page_tlb_size") { - new_table_options->huge_page_tlb_size = ParseSizeT(o.second); - } else if (o.first == "encoding_type") { - if (o.second == "kPlain") { - new_table_options->encoding_type = kPlain; - } else if (o.second == "kPrefix") { - new_table_options->encoding_type = kPrefix; - } else { - throw std::invalid_argument("Unknown encoding_type: " + o.second); - } - } else if (o.first == "full_scan_mode") { - new_table_options->full_scan_mode = ParseBoolean(o.first, o.second); - } else if (o.first == "store_index_in_file") { - new_table_options->store_index_in_file = - ParseBoolean(o.first, o.second); - } else { - return Status::InvalidArgument("Unrecognized option: " + o.first); + auto error_message = ParsePlainTableOptions( + o.first, o.second, new_table_options, input_strings_escaped); + if (error_message != "") { + const auto iter = plain_table_type_info.find(o.first); + if (iter == plain_table_type_info.end() || + !input_strings_escaped ||// !input_strings_escaped indicates + // the old API, where everything is + // parsable. + (iter->second.verification != OptionVerificationType::kByName && + iter->second.verification != OptionVerificationType::kDeprecated)) { + return Status::InvalidArgument("Can't parse PlainTableOptions:", + o.first + " " + error_message); } - } catch (std::exception& e) { - return Status::InvalidArgument("error parsing " + o.first + ":" + - std::string(e.what())); } } return Status::OK(); } +Status GetPlainTableOptionsFromString( + const PlainTableOptions& table_options, + const std::string& opts_str, + PlainTableOptions* new_table_options) { + std::unordered_map opts_map; + Status s = StringToMap(opts_str, &opts_map); + if (!s.ok()) { + return s; + } + return GetPlainTableOptionsFromMap(table_options, opts_map, + new_table_options); +} + Status GetColumnFamilyOptionsFromMap( const ColumnFamilyOptions& base_options, const std::unordered_map& opts_map, @@ -1239,6 +1299,15 @@ Status GetTableFactoryFromMap( } table_factory->reset(new BlockBasedTableFactory(bbt_opt)); return Status::OK(); + } else if (factory_name == PlainTableFactory().Name()) { + PlainTableOptions pt_opt; + s = GetPlainTableOptionsFromMap(PlainTableOptions(), opt_map, + &pt_opt, true); + if (!s.ok()) { + return s; + } + table_factory->reset(new PlainTableFactory(pt_opt)); + return Status::OK(); } // Return OK for not supported table factories as TableFactory // Deserialization is optional. diff --git a/util/options_helper.h b/util/options_helper.h index 3f6eab40a..22a34c99c 100644 --- a/util/options_helper.h +++ b/util/options_helper.h @@ -87,6 +87,7 @@ enum class OptionType { kFilterPolicy, kFlushBlockPolicyFactory, kChecksumType, + kEncodingType, kUnknown }; @@ -459,6 +460,34 @@ static std::unordered_map plain_table_type_info = { + {"user_key_len", + {offsetof(struct PlainTableOptions, user_key_len), + OptionType::kUInt32T, OptionVerificationType::kNormal}}, + {"bloom_bits_per_key", + {offsetof(struct PlainTableOptions, bloom_bits_per_key), + OptionType::kInt, OptionVerificationType::kNormal}}, + {"hash_table_ratio", + {offsetof(struct PlainTableOptions, hash_table_ratio), + OptionType::kDouble, OptionVerificationType::kNormal}}, + {"index_sparseness", + {offsetof(struct PlainTableOptions, index_sparseness), + OptionType::kSizeT, OptionVerificationType::kNormal}}, + {"huge_page_tlb_size", + {offsetof(struct PlainTableOptions, huge_page_tlb_size), + OptionType::kSizeT, OptionVerificationType::kNormal}}, + {"encoding_type", + {offsetof(struct PlainTableOptions, encoding_type), + OptionType::kEncodingType, OptionVerificationType::kByName}}, + {"full_scan_mode", + {offsetof(struct PlainTableOptions, full_scan_mode), + OptionType::kBoolean, OptionVerificationType::kNormal}}, + {"store_index_in_file", + {offsetof(struct PlainTableOptions, store_index_in_file), + OptionType::kBoolean, OptionVerificationType::kNormal}}}; + } // namespace rocksdb #endif // !ROCKSDB_LITE diff --git a/util/options_test.cc b/util/options_test.cc index e1849bb3f..59906027b 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -23,6 +23,7 @@ #include "rocksdb/table.h" #include "rocksdb/utilities/leveldb_options.h" #include "table/block_based_table_factory.h" +#include "table/plain_table_factory.h" #include "util/options_helper.h" #include "util/options_parser.h" #include "util/random.h" @@ -588,6 +589,23 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { ASSERT_NOK(GetColumnFamilyOptionsFromString(base_cf_opt, "optimize_filters_for_hits=junk", &new_cf_opt)); + + // Nested plain table options + // Emtpy + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={};arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(new_cf_opt.table_factory->Name(), "PlainTable"); + // Non-empty + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "plain_table_factory={user_key_len=66;bloom_bits_per_key=20;};" + "arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.table_factory != nullptr); + ASSERT_EQ(new_cf_opt.table_factory->Name(), "PlainTable"); } #endif // !ROCKSDB_LITE @@ -646,6 +664,40 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) { } #endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetPlainTableOptionsFromString is not supported +TEST_F(OptionsTest, GetPlainTableOptionsFromString) { + PlainTableOptions table_opt; + PlainTableOptions new_opt; + // make sure default values are overwritten by something else + ASSERT_OK(GetPlainTableOptionsFromString(table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "index_sparseness=8;huge_page_tlb_size=4;encoding_type=kPrefix;" + "full_scan_mode=true;store_index_in_file=true", + &new_opt)); + ASSERT_EQ(new_opt.user_key_len, 66); + ASSERT_EQ(new_opt.bloom_bits_per_key, 20); + ASSERT_EQ(new_opt.hash_table_ratio, 0.5); + ASSERT_EQ(new_opt.index_sparseness, 8); + ASSERT_EQ(new_opt.huge_page_tlb_size, 4); + ASSERT_EQ(new_opt.encoding_type, EncodingType::kPrefix); + ASSERT_TRUE(new_opt.full_scan_mode); + ASSERT_TRUE(new_opt.store_index_in_file); + + // unknown option + ASSERT_NOK(GetPlainTableOptionsFromString(table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "bad_option=1", + &new_opt)); + + // unrecognized EncodingType + ASSERT_NOK(GetPlainTableOptionsFromString(table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "encoding_type=kPrefixXX", + &new_opt)); +} +#endif // !ROCKSDB_LITE + #ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite TEST_F(OptionsTest, GetOptionsFromStringTest) { Options base_options, new_options;