DBOptions serialization and deserialization

Summary:
This patch implements DBOptions deserialization and improve
the current implementation of DBOptions serialization by
using a static structure that stores the offset of each
DBOptions member variables to perform serialization and
deserialization instead of using tons of if-then-branch
to determine the mapping between string and variables.

Test Plan: Added test in options_test.cc

Reviewers: igor, anthony, sdong, IslamAbdelRahman

Reviewed By: sdong

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D44097
main
Yueh-Hsuan Chiang 9 years ago
parent b2df20a890
commit 9ec9571593
  1. 3
      include/rocksdb/convenience.h
  2. 160
      util/options_helper.cc
  3. 117
      util/options_helper.h
  4. 131
      util/options_test.cc

@ -48,6 +48,9 @@ Status GetDBOptionsFromString(
const std::string& opts_str, const std::string& opts_str,
DBOptions* new_options); DBOptions* new_options);
Status GetStringFromDBOptions(const DBOptions& db_options,
std::string* opts_str);
Status GetBlockBasedTableOptionsFromString( Status GetBlockBasedTableOptionsFromString(
const BlockBasedTableOptions& table_options, const BlockBasedTableOptions& table_options,
const std::string& opts_str, const std::string& opts_str,

@ -17,6 +17,7 @@
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "table/block_based_table_factory.h" #include "table/block_based_table_factory.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/string_util.h"
namespace rocksdb { namespace rocksdb {
@ -483,83 +484,104 @@ bool ParseColumnFamilyOption(const std::string& name, const std::string& value,
return true; return true;
} }
bool SerializeSingleDBOption(const DBOptions& db_options,
const std::string& name, std::string* opt_string) {
auto iter = db_options_type_info.find(name);
if (iter == db_options_type_info.end()) {
return false;
}
auto& opt_info = iter->second;
const char* opt_address =
reinterpret_cast<const char*>(&db_options) + opt_info.offset;
std::string value;
switch (opt_info.type) {
case OptionType::kBoolean:
value = *(reinterpret_cast<const bool*>(opt_address)) ? "true" : "false";
break;
case OptionType::kInt:
value = ToString(*(reinterpret_cast<const int*>(opt_address)));
break;
case OptionType::kUInt:
value = ToString(*(reinterpret_cast<const unsigned int*>(opt_address)));
break;
case OptionType::kUInt32T:
value = ToString(*(reinterpret_cast<const uint32_t*>(opt_address)));
break;
case OptionType::kUInt64T:
value = ToString(*(reinterpret_cast<const uint64_t*>(opt_address)));
break;
case OptionType::kSizeT:
value = ToString(*(reinterpret_cast<const size_t*>(opt_address)));
break;
case OptionType::kString:
value = *(reinterpret_cast<const std::string*>(opt_address));
break;
default:
return false;
}
*opt_string = name + " = " + value + "; ";
return true;
}
Status GetStringFromDBOptions(const DBOptions& db_options,
std::string* opt_string) {
assert(opt_string);
opt_string->clear();
for (auto iter = db_options_type_info.begin();
iter != db_options_type_info.end(); ++iter) {
std::string single_output;
bool result =
SerializeSingleDBOption(db_options, iter->first, &single_output);
assert(result);
if (result) {
opt_string->append(single_output);
}
}
return Status::OK();
}
bool ParseDBOption(const std::string& name, const std::string& value, bool ParseDBOption(const std::string& name, const std::string& value,
DBOptions* new_options) { DBOptions* new_options) {
try { try {
if (name == "create_if_missing") { if (name == "rate_limiter_bytes_per_sec") {
new_options->create_if_missing = ParseBoolean(name, value);
} else if (name == "create_missing_column_families") {
new_options->create_missing_column_families =
ParseBoolean(name, value);
} else if (name == "error_if_exists") {
new_options->error_if_exists = ParseBoolean(name, value);
} else if (name == "paranoid_checks") {
new_options->paranoid_checks = ParseBoolean(name, value);
} else if (name == "rate_limiter_bytes_per_sec") {
new_options->rate_limiter.reset( new_options->rate_limiter.reset(
NewGenericRateLimiter(static_cast<int64_t>(ParseUint64(value)))); NewGenericRateLimiter(static_cast<int64_t>(ParseUint64(value))));
} else if (name == "max_open_files") {
new_options->max_open_files = ParseInt(value);
} else if (name == "max_total_wal_size") {
new_options->max_total_wal_size = ParseUint64(value);
} else if (name == "disable_data_sync") {
new_options->disableDataSync = ParseBoolean(name, value);
} else if (name == "use_fsync") {
new_options->use_fsync = ParseBoolean(name, value);
} else if (name == "db_paths") {
// TODO(ljin): add support
return false;
} else if (name == "db_log_dir") {
new_options->db_log_dir = value;
} else if (name == "wal_dir") {
new_options->wal_dir = value;
} else if (name == "delete_obsolete_files_period_micros") {
new_options->delete_obsolete_files_period_micros = ParseUint64(value);
} else if (name == "max_background_compactions") {
new_options->max_background_compactions = ParseInt(value);
} else if (name == "max_background_flushes") {
new_options->max_background_flushes = ParseInt(value);
} else if (name == "max_log_file_size") {
new_options->max_log_file_size = ParseSizeT(value);
} else if (name == "log_file_time_to_roll") {
new_options->log_file_time_to_roll = ParseSizeT(value);
} else if (name == "keep_log_file_num") {
new_options->keep_log_file_num = ParseSizeT(value);
} else if (name == "max_manifest_file_size") {
new_options->max_manifest_file_size = ParseUint64(value);
} else if (name == "table_cache_numshardbits") {
new_options->table_cache_numshardbits = ParseInt(value);
} else if (name == "WAL_ttl_seconds") {
new_options->WAL_ttl_seconds = ParseUint64(value);
} else if (name == "WAL_size_limit_MB") {
new_options->WAL_size_limit_MB = ParseUint64(value);
} else if (name == "manifest_preallocation_size") {
new_options->manifest_preallocation_size = ParseSizeT(value);
} else if (name == "allow_os_buffer") {
new_options->allow_os_buffer = ParseBoolean(name, value);
} else if (name == "allow_mmap_reads") {
new_options->allow_mmap_reads = ParseBoolean(name, value);
} else if (name == "allow_mmap_writes") {
new_options->allow_mmap_writes = ParseBoolean(name, value);
} else if (name == "is_fd_close_on_exec") {
new_options->is_fd_close_on_exec = ParseBoolean(name, value);
} else if (name == "skip_log_error_on_recovery") {
new_options->skip_log_error_on_recovery = ParseBoolean(name, value);
} else if (name == "stats_dump_period_sec") {
new_options->stats_dump_period_sec = ParseUint32(value);
} else if (name == "advise_random_on_open") {
new_options->advise_random_on_open = ParseBoolean(name, value);
} else if (name == "db_write_buffer_size") {
new_options->db_write_buffer_size = ParseUint64(value);
} else if (name == "use_adaptive_mutex") {
new_options->use_adaptive_mutex = ParseBoolean(name, value);
} else if (name == "bytes_per_sync") {
new_options->bytes_per_sync = ParseUint64(value);
} else if (name == "wal_bytes_per_sync") {
new_options->wal_bytes_per_sync = ParseUint64(value);
} else { } else {
auto iter = db_options_type_info.find(name);
if (iter == db_options_type_info.end()) {
return false; return false;
} }
auto& opt_info = iter->second;
char* opt_address =
reinterpret_cast<char*>(new_options) + opt_info.offset;
switch (opt_info.type) {
case OptionType::kBoolean:
*reinterpret_cast<bool*>(opt_address) = ParseBoolean("", value);
break;
case OptionType::kInt:
*reinterpret_cast<int*>(opt_address) = ParseInt(value);
break;
case OptionType::kUInt:
*reinterpret_cast<unsigned int*>(opt_address) = ParseUint32(value);
break;
case OptionType::kUInt32T:
*reinterpret_cast<uint32_t*>(opt_address) = ParseUint32(value);
break;
case OptionType::kUInt64T:
*reinterpret_cast<uint64_t*>(opt_address) = ParseUint64(value);
break;
case OptionType::kSizeT:
*reinterpret_cast<size_t*>(opt_address) = ParseUint32(value);
break;
case OptionType::kString:
*reinterpret_cast<std::string*>(opt_address) = value;
break;
default:
return false;
}
}
} catch (const std::exception& e) { } catch (const std::exception& e) {
return false; return false;
} }

@ -17,4 +17,121 @@ Status GetMutableOptionsFromStrings(
const std::unordered_map<std::string, std::string>& options_map, const std::unordered_map<std::string, std::string>& options_map,
MutableCFOptions* new_options); MutableCFOptions* new_options);
enum class OptionType {
kBoolean,
kInt,
kUInt,
kUInt32T,
kUInt64T,
kSizeT,
kString,
kUnknown
};
// A struct for storing constant option information such as option name,
// option type, and offset.
struct OptionTypeInfo {
int offset;
OptionType type;
};
static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
/*
// not yet supported
AccessHint access_hint_on_compaction_start;
Env* env;
InfoLogLevel info_log_level;
WALRecoveryMode wal_recovery_mode;
std::shared_ptr<Cache> row_cache;
std::shared_ptr<DeleteScheduler> delete_scheduler;
std::shared_ptr<Logger> info_log;
std::shared_ptr<RateLimiter> rate_limiter;
std::shared_ptr<Statistics> statistics;
std::vector<DbPath> db_paths;
std::vector<std::shared_ptr<EventListener>> listeners;
*/
{"advise_random_on_open",
{offsetof(struct DBOptions, advise_random_on_open), OptionType::kBoolean}},
{"allow_mmap_reads",
{offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean}},
{"allow_mmap_writes",
{offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean}},
{"allow_os_buffer",
{offsetof(struct DBOptions, allow_os_buffer), OptionType::kBoolean}},
{"create_if_missing",
{offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean}},
{"create_missing_column_families",
{offsetof(struct DBOptions, create_missing_column_families),
OptionType::kBoolean}},
{"disableDataSync",
{offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}},
{"disable_data_sync", // for compatibility
{offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}},
{"enable_thread_tracking",
{offsetof(struct DBOptions, enable_thread_tracking),
OptionType::kBoolean}},
{"error_if_exists",
{offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean}},
{"is_fd_close_on_exec",
{offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean}},
{"paranoid_checks",
{offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean}},
{"skip_log_error_on_recovery",
{offsetof(struct DBOptions, skip_log_error_on_recovery),
OptionType::kBoolean}},
{"skip_stats_update_on_db_open",
{offsetof(struct DBOptions, skip_stats_update_on_db_open),
OptionType::kBoolean}},
{"use_adaptive_mutex",
{offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean}},
{"use_fsync",
{offsetof(struct DBOptions, use_fsync), OptionType::kBoolean}},
{"max_background_compactions",
{offsetof(struct DBOptions, max_background_compactions),
OptionType::kInt}},
{"max_background_flushes",
{offsetof(struct DBOptions, max_background_flushes), OptionType::kInt}},
{"max_file_opening_threads",
{offsetof(struct DBOptions, max_file_opening_threads), OptionType::kInt}},
{"max_open_files",
{offsetof(struct DBOptions, max_open_files), OptionType::kInt}},
{"table_cache_numshardbits",
{offsetof(struct DBOptions, table_cache_numshardbits), OptionType::kInt}},
{"db_write_buffer_size",
{offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT}},
{"keep_log_file_num",
{offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT}},
{"log_file_time_to_roll",
{offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT}},
{"manifest_preallocation_size",
{offsetof(struct DBOptions, manifest_preallocation_size),
OptionType::kSizeT}},
{"max_log_file_size",
{offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT}},
{"db_log_dir",
{offsetof(struct DBOptions, db_log_dir), OptionType::kString}},
{"wal_dir", {offsetof(struct DBOptions, wal_dir), OptionType::kString}},
{"num_subcompactions",
{offsetof(struct DBOptions, num_subcompactions), OptionType::kUInt32T}},
{"WAL_size_limit_MB",
{offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T}},
{"WAL_ttl_seconds",
{offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T}},
{"bytes_per_sync",
{offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T}},
{"delayed_write_rate",
{offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T}},
{"delete_obsolete_files_period_micros",
{offsetof(struct DBOptions, delete_obsolete_files_period_micros),
OptionType::kUInt64T}},
{"max_manifest_file_size",
{offsetof(struct DBOptions, max_manifest_file_size),
OptionType::kUInt64T}},
{"max_total_wal_size",
{offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T}},
{"wal_bytes_per_sync",
{offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T}},
{"stats_dump_period_sec",
{offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt}}};
} // namespace rocksdb } // namespace rocksdb

@ -504,6 +504,137 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) {
ASSERT_EQ(new_options.max_open_files, 1); ASSERT_EQ(new_options.max_open_files, 1);
ASSERT_TRUE(new_options.rate_limiter.get() != nullptr); ASSERT_TRUE(new_options.rate_limiter.get() != nullptr);
} }
namespace {
void VerifyDBOptions(const DBOptions& base_opt, const DBOptions& new_opt) {
// boolean options
ASSERT_EQ(base_opt.advise_random_on_open, new_opt.advise_random_on_open);
ASSERT_EQ(base_opt.allow_mmap_reads, new_opt.allow_mmap_reads);
ASSERT_EQ(base_opt.allow_mmap_writes, new_opt.allow_mmap_writes);
ASSERT_EQ(base_opt.allow_os_buffer, new_opt.allow_os_buffer);
ASSERT_EQ(base_opt.create_if_missing, new_opt.create_if_missing);
ASSERT_EQ(base_opt.create_missing_column_families,
new_opt.create_missing_column_families);
ASSERT_EQ(base_opt.disableDataSync, new_opt.disableDataSync);
ASSERT_EQ(base_opt.enable_thread_tracking, new_opt.enable_thread_tracking);
ASSERT_EQ(base_opt.error_if_exists, new_opt.error_if_exists);
ASSERT_EQ(base_opt.is_fd_close_on_exec, new_opt.is_fd_close_on_exec);
ASSERT_EQ(base_opt.paranoid_checks, new_opt.paranoid_checks);
ASSERT_EQ(base_opt.skip_log_error_on_recovery,
new_opt.skip_log_error_on_recovery);
ASSERT_EQ(base_opt.skip_stats_update_on_db_open,
new_opt.skip_stats_update_on_db_open);
ASSERT_EQ(base_opt.use_adaptive_mutex, new_opt.use_adaptive_mutex);
ASSERT_EQ(base_opt.use_fsync, new_opt.use_fsync);
// int options
ASSERT_EQ(base_opt.max_background_compactions,
new_opt.max_background_compactions);
ASSERT_EQ(base_opt.max_background_flushes, new_opt.max_background_flushes);
ASSERT_EQ(base_opt.max_file_opening_threads,
new_opt.max_file_opening_threads);
ASSERT_EQ(base_opt.max_open_files, new_opt.max_open_files);
ASSERT_EQ(base_opt.table_cache_numshardbits,
new_opt.table_cache_numshardbits);
// size_t options
ASSERT_EQ(base_opt.db_write_buffer_size, new_opt.db_write_buffer_size);
ASSERT_EQ(base_opt.keep_log_file_num, new_opt.keep_log_file_num);
ASSERT_EQ(base_opt.log_file_time_to_roll, new_opt.log_file_time_to_roll);
ASSERT_EQ(base_opt.manifest_preallocation_size,
new_opt.manifest_preallocation_size);
ASSERT_EQ(base_opt.max_log_file_size, new_opt.max_log_file_size);
// std::string options
ASSERT_EQ(base_opt.db_log_dir, new_opt.db_log_dir);
ASSERT_EQ(base_opt.wal_dir, new_opt.wal_dir);
// uint32_t options
ASSERT_EQ(base_opt.num_subcompactions, new_opt.num_subcompactions);
// uint64_t options
ASSERT_EQ(base_opt.WAL_size_limit_MB, new_opt.WAL_size_limit_MB);
ASSERT_EQ(base_opt.WAL_ttl_seconds, new_opt.WAL_ttl_seconds);
ASSERT_EQ(base_opt.bytes_per_sync, new_opt.bytes_per_sync);
ASSERT_EQ(base_opt.delayed_write_rate, new_opt.delayed_write_rate);
ASSERT_EQ(base_opt.delete_obsolete_files_period_micros,
new_opt.delete_obsolete_files_period_micros);
ASSERT_EQ(base_opt.max_manifest_file_size, new_opt.max_manifest_file_size);
ASSERT_EQ(base_opt.max_total_wal_size, new_opt.max_total_wal_size);
ASSERT_EQ(base_opt.wal_bytes_per_sync, new_opt.wal_bytes_per_sync);
// unsigned int options
ASSERT_EQ(base_opt.stats_dump_period_sec, new_opt.stats_dump_period_sec);
}
} // namespace
TEST_F(OptionsTest, DBOptionsSerialization) {
Options base_options, new_options;
Random rnd(301);
// Phase 1: Make big change in base_options
// boolean options
base_options.advise_random_on_open = rnd.Uniform(2);
base_options.allow_mmap_reads = rnd.Uniform(2);
base_options.allow_mmap_writes = rnd.Uniform(2);
base_options.allow_os_buffer = rnd.Uniform(2);
base_options.create_if_missing = rnd.Uniform(2);
base_options.create_missing_column_families = rnd.Uniform(2);
base_options.disableDataSync = rnd.Uniform(2);
base_options.enable_thread_tracking = rnd.Uniform(2);
base_options.error_if_exists = rnd.Uniform(2);
base_options.is_fd_close_on_exec = rnd.Uniform(2);
base_options.paranoid_checks = rnd.Uniform(2);
base_options.skip_log_error_on_recovery = rnd.Uniform(2);
base_options.skip_stats_update_on_db_open = rnd.Uniform(2);
base_options.use_adaptive_mutex = rnd.Uniform(2);
base_options.use_fsync = rnd.Uniform(2);
// int options
base_options.max_background_compactions = rnd.Uniform(100);
base_options.max_background_flushes = rnd.Uniform(100);
base_options.max_file_opening_threads = rnd.Uniform(100);
base_options.max_open_files = rnd.Uniform(100);
base_options.table_cache_numshardbits = rnd.Uniform(100);
// size_t options
base_options.db_write_buffer_size = rnd.Uniform(10000);
base_options.keep_log_file_num = rnd.Uniform(10000);
base_options.log_file_time_to_roll = rnd.Uniform(10000);
base_options.manifest_preallocation_size = rnd.Uniform(10000);
base_options.max_log_file_size = rnd.Uniform(10000);
// std::string options
base_options.db_log_dir = "path/to/db_log_dir";
base_options.wal_dir = "path/to/wal_dir";
// uint32_t options
base_options.num_subcompactions = rnd.Uniform(100000);
// uint64_t options
static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
base_options.WAL_size_limit_MB = uint_max + rnd.Uniform(100000);
base_options.WAL_ttl_seconds = uint_max + rnd.Uniform(100000);
base_options.bytes_per_sync = uint_max + rnd.Uniform(100000);
base_options.delayed_write_rate = uint_max + rnd.Uniform(100000);
base_options.delete_obsolete_files_period_micros =
uint_max + rnd.Uniform(100000);
base_options.max_manifest_file_size = uint_max + rnd.Uniform(100000);
base_options.max_total_wal_size = uint_max + rnd.Uniform(100000);
base_options.wal_bytes_per_sync = uint_max + rnd.Uniform(100000);
// unsigned int options
base_options.stats_dump_period_sec = rnd.Uniform(100000);
// Phase 2: obtain a string from base_option
std::string base_opt_string;
ASSERT_OK(GetStringFromDBOptions(base_options, &base_opt_string));
// Phase 3: Set new_options from the derived string and expect
// new_options == base_options
ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_opt_string, &new_options));
VerifyDBOptions(base_options, new_options);
}
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE

Loading…
Cancel
Save