From bd7be035e04ca2844f8627910d369626660ec56c Mon Sep 17 00:00:00 2001 From: SherlockNoMad Date: Tue, 17 Nov 2015 14:29:01 -0800 Subject: [PATCH] Support Memtable Factory Parse in option_helper.cc --- include/rocksdb/convenience.h | 4 ++ util/options_helper.cc | 79 +++++++++++++++++++++++++++++++++++ util/options_test.cc | 57 ++++++++++++++++++++++--- 3 files changed, 135 insertions(+), 5 deletions(-) diff --git a/include/rocksdb/convenience.h b/include/rocksdb/convenience.h index 15df53204..7a65b1388 100644 --- a/include/rocksdb/convenience.h +++ b/include/rocksdb/convenience.h @@ -85,6 +85,10 @@ Status GetPlainTableOptionsFromString( const std::string& opts_str, PlainTableOptions* new_table_options); +Status GetMemTableRepFactoryFromString( + const std::string& opts_str, + std::unique_ptr* new_mem_factory); + Status GetOptionsFromString(const Options& base_options, const std::string& opts_str, Options* new_options); diff --git a/util/options_helper.cc b/util/options_helper.cc index 2957bd5d5..1cb70ff08 100644 --- a/util/options_helper.cc +++ b/util/options_helper.cc @@ -865,6 +865,15 @@ Status ParseColumnFamilyOption(const std::string& name, "unable to parse the specified CF option " + name); } new_options->table_factory.reset(NewPlainTableFactory(table_opt)); + } else if (name == "memtable") { + std::unique_ptr new_mem_factory; + Status mem_factory_s = + GetMemTableRepFactoryFromString(value, &new_mem_factory); + if (!mem_factory_s.ok()) { + return Status::InvalidArgument( + "unable to parse the specified CF option " + name); + } + new_options->memtable_factory.reset(new_mem_factory.release()); } else if (name == "compression_opts") { size_t start = 0; size_t end = value.find(':'); @@ -1243,6 +1252,76 @@ Status GetPlainTableOptionsFromString( new_table_options); } +Status GetMemTableRepFactoryFromString(const std::string& opts_str, + std::unique_ptr* new_mem_factory) { + std::vector opts_list = StringSplit(opts_str, ':'); + size_t len = opts_list.size(); + + if (opts_list.size() <= 0 || opts_list.size() > 2) { + return Status::InvalidArgument("Can't parse memtable_factory option ", + opts_str); + } + + MemTableRepFactory* mem_factory = nullptr; + + if (opts_list[0] == "skip_list") { + // Expecting format + // skip_list: + if (2 == len) { + size_t lookahead = ParseSizeT(opts_list[1]); + mem_factory = new SkipListFactory(lookahead); + } else if (1 == len) { + mem_factory = new SkipListFactory(); + } + } else if (opts_list[0] == "prefix_hash") { + // Expecting format + // prfix_hash: + if (2 == len) { + size_t hash_bucket_count = ParseSizeT(opts_list[1]); + mem_factory = NewHashSkipListRepFactory(hash_bucket_count); + } else if (1 == len) { + mem_factory = NewHashSkipListRepFactory(); + } + } else if (opts_list[0] == "hash_linkedlist") { + // Expecting format + // hash_linkedlist: + if (2 == len) { + size_t hash_bucket_count = ParseSizeT(opts_list[1]); + mem_factory = NewHashLinkListRepFactory(hash_bucket_count); + } else if (1 == len) { + mem_factory = NewHashLinkListRepFactory(); + } + } else if (opts_list[0] == "vector") { + // Expecting format + // vector: + if (2 == len) { + size_t count = ParseSizeT(opts_list[1]); + mem_factory = new VectorRepFactory(count); + } else if (1 == len) { + mem_factory = new VectorRepFactory(); + } + } else if (opts_list[0] == "cuckoo") { + // Expecting format + // cuckoo: + if (2 == len) { + size_t write_buffer_size = ParseSizeT(opts_list[1]); + mem_factory= NewHashCuckooRepFactory(write_buffer_size); + } else if (1 == len) { + return Status::InvalidArgument("Can't parse memtable_factory option ", + opts_str); + } + } else { + return Status::InvalidArgument("Unrecognized memtable_factory option ", + opts_str); + } + + if (mem_factory != nullptr){ + new_mem_factory->reset(mem_factory); + } + + return Status::OK(); +} + Status GetColumnFamilyOptionsFromMap( const ColumnFamilyOptions& base_options, const std::unordered_map& opts_map, diff --git a/util/options_test.cc b/util/options_test.cc index 49d4851e2..2ecce083c 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -17,6 +17,7 @@ #include "rocksdb/cache.h" #include "rocksdb/convenience.h" +#include "rocksdb/memtablerep.h" #include "rocksdb/utilities/leveldb_options.h" #include "util/options_helper.h" #include "util/options_parser.h" @@ -448,6 +449,14 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { &new_cf_opt)); ASSERT_TRUE(new_cf_opt.table_factory != nullptr); ASSERT_EQ(std::string(new_cf_opt.table_factory->Name()), "PlainTable"); + + // memtable factory + ASSERT_OK(GetColumnFamilyOptionsFromString(base_cf_opt, + "write_buffer_size=10;max_write_buffer_number=16;" + "memtable=skip_list:10;arena_block_size=1024", + &new_cf_opt)); + ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr); + ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory"); } #endif // !ROCKSDB_LITE @@ -535,11 +544,49 @@ TEST_F(OptionsTest, GetPlainTableOptionsFromString) { &new_opt)); // unrecognized EncodingType - ASSERT_NOK(GetPlainTableOptionsFromString( - table_opt, - "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" - "encoding_type=kPrefixXX", - &new_opt)); + ASSERT_NOK(GetPlainTableOptionsFromString(table_opt, + "user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;" + "encoding_type=kPrefixXX", + &new_opt)); +} +#endif // !ROCKSDB_LITE + +#ifndef ROCKSDB_LITE // GetMemTableRepFactoryFromString is not supported +TEST_F(OptionsTest, GetMemTableRepFactoryFromString) { + std::unique_ptr new_mem_factory = nullptr; + + ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000", + &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist", + &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("hash_linkedlist:1000", + &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "HashLinkListRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("hash_linkedlist:1000:invalid_opt", + &new_mem_factory)); + + ASSERT_OK(GetMemTableRepFactoryFromString("vector", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("vector:1024", &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "VectorRepFactory"); + ASSERT_NOK(GetMemTableRepFactoryFromString("vector:1024:invalid_opt", + &new_mem_factory)); + + ASSERT_NOK(GetMemTableRepFactoryFromString("cuckoo", &new_mem_factory)); + ASSERT_OK(GetMemTableRepFactoryFromString("cuckoo:1024", &new_mem_factory)); + ASSERT_EQ(std::string(new_mem_factory->Name()), "HashCuckooRepFactory"); + + ASSERT_NOK(GetMemTableRepFactoryFromString("bad_factory", &new_mem_factory)); } #endif // !ROCKSDB_LITE