DBOptions serialization and deserialization

Summary:
This patch implements DBOptions deserialization and improve
the current implementation of DBOptions serialization by
using a static structure that stores the offset of each
DBOptions member variables to perform serialization and
deserialization instead of using tons of if-then-branch
to determine the mapping between string and variables.

Test Plan: Added test in options_test.cc

Reviewers: igor, anthony, sdong, IslamAbdelRahman

Reviewed By: sdong

Subscribers: dhruba

Differential Revision: https://reviews.facebook.net/D44097

include/rocksdb/convenience.h

@@ -48,6 +48,9 @@ Status GetDBOptionsFromString(
     const std::string& opts_str,
     DBOptions* new_options);
 
+Status GetStringFromDBOptions(const DBOptions& db_options,
+                              std::string* opts_str);
+
 Status GetBlockBasedTableOptionsFromString(
     const BlockBasedTableOptions& table_options,
     const std::string& opts_str,

util/options_helper.cc

@@ -17,6 +17,7 @@
 #include "rocksdb/table.h"
 #include "table/block_based_table_factory.h"
 #include "util/logging.h"
+#include "util/string_util.h"
 
 namespace rocksdb {
 
@@ -483,82 +484,103 @@ bool ParseColumnFamilyOption(const std::string& name, const std::string& value,
   return true;
 }
 
+bool SerializeSingleDBOption(const DBOptions& db_options,
+                             const std::string& name, std::string* opt_string) {
+  auto iter = db_options_type_info.find(name);
+  if (iter == db_options_type_info.end()) {
+    return false;
+  }
+  auto& opt_info = iter->second;
+  const char* opt_address =
+      reinterpret_cast<const char*>(&db_options) + opt_info.offset;
+  std::string value;
+
+  switch (opt_info.type) {
+    case OptionType::kBoolean:
+      value = *(reinterpret_cast<const bool*>(opt_address)) ? "true" : "false";
+      break;
+    case OptionType::kInt:
+      value = ToString(*(reinterpret_cast<const int*>(opt_address)));
+      break;
+    case OptionType::kUInt:
+      value = ToString(*(reinterpret_cast<const unsigned int*>(opt_address)));
+      break;
+    case OptionType::kUInt32T:
+      value = ToString(*(reinterpret_cast<const uint32_t*>(opt_address)));
+      break;
+    case OptionType::kUInt64T:
+      value = ToString(*(reinterpret_cast<const uint64_t*>(opt_address)));
+      break;
+    case OptionType::kSizeT:
+      value = ToString(*(reinterpret_cast<const size_t*>(opt_address)));
+      break;
+    case OptionType::kString:
+      value = *(reinterpret_cast<const std::string*>(opt_address));
+      break;
+    default:
+      return false;
+  }
+
+  *opt_string = name + " = " + value + ";  ";
+  return true;
+}
+
+Status GetStringFromDBOptions(const DBOptions& db_options,
+                              std::string* opt_string) {
+  assert(opt_string);
+  opt_string->clear();
+  for (auto iter = db_options_type_info.begin();
+       iter != db_options_type_info.end(); ++iter) {
+    std::string single_output;
+    bool result =
+        SerializeSingleDBOption(db_options, iter->first, &single_output);
+    assert(result);
+    if (result) {
+      opt_string->append(single_output);
+    }
+  }
+  return Status::OK();
+}
+
 bool ParseDBOption(const std::string& name, const std::string& value,
                    DBOptions* new_options) {
   try {
-    if (name == "create_if_missing") {
+    if (name == "rate_limiter_bytes_per_sec") {
-      new_options->create_if_missing = ParseBoolean(name, value);
-    } else if (name == "create_missing_column_families") {
-      new_options->create_missing_column_families =
-          ParseBoolean(name, value);
-    } else if (name == "error_if_exists") {
-      new_options->error_if_exists = ParseBoolean(name, value);
-    } else if (name == "paranoid_checks") {
-      new_options->paranoid_checks = ParseBoolean(name, value);
-    } else if (name == "rate_limiter_bytes_per_sec") {
       new_options->rate_limiter.reset(
           NewGenericRateLimiter(static_cast<int64_t>(ParseUint64(value))));
-    } else if (name == "max_open_files") {
-      new_options->max_open_files = ParseInt(value);
-    } else if (name == "max_total_wal_size") {
-      new_options->max_total_wal_size = ParseUint64(value);
-    } else if (name == "disable_data_sync") {
-      new_options->disableDataSync = ParseBoolean(name, value);
-    } else if (name == "use_fsync") {
-      new_options->use_fsync = ParseBoolean(name, value);
-    } else if (name == "db_paths") {
-      // TODO(ljin): add support
-      return false;
-    } else if (name == "db_log_dir") {
-      new_options->db_log_dir = value;
-    } else if (name == "wal_dir") {
-      new_options->wal_dir = value;
-    } else if (name == "delete_obsolete_files_period_micros") {
-      new_options->delete_obsolete_files_period_micros = ParseUint64(value);
-    } else if (name == "max_background_compactions") {
-      new_options->max_background_compactions = ParseInt(value);
-    } else if (name == "max_background_flushes") {
-      new_options->max_background_flushes = ParseInt(value);
-    } else if (name == "max_log_file_size") {
-      new_options->max_log_file_size = ParseSizeT(value);
-    } else if (name == "log_file_time_to_roll") {
-      new_options->log_file_time_to_roll = ParseSizeT(value);
-    } else if (name == "keep_log_file_num") {
-      new_options->keep_log_file_num = ParseSizeT(value);
-    } else if (name == "max_manifest_file_size") {
-      new_options->max_manifest_file_size = ParseUint64(value);
-    } else if (name == "table_cache_numshardbits") {
-      new_options->table_cache_numshardbits = ParseInt(value);
-    } else if (name == "WAL_ttl_seconds") {
-      new_options->WAL_ttl_seconds = ParseUint64(value);
-    } else if (name == "WAL_size_limit_MB") {
-      new_options->WAL_size_limit_MB = ParseUint64(value);
-    } else if (name == "manifest_preallocation_size") {
-      new_options->manifest_preallocation_size = ParseSizeT(value);
-    } else if (name == "allow_os_buffer") {
-      new_options->allow_os_buffer = ParseBoolean(name, value);
-    } else if (name == "allow_mmap_reads") {
-      new_options->allow_mmap_reads = ParseBoolean(name, value);
-    } else if (name == "allow_mmap_writes") {
-      new_options->allow_mmap_writes = ParseBoolean(name, value);
-    } else if (name == "is_fd_close_on_exec") {
-      new_options->is_fd_close_on_exec = ParseBoolean(name, value);
-    } else if (name == "skip_log_error_on_recovery") {
-      new_options->skip_log_error_on_recovery = ParseBoolean(name, value);
-    } else if (name == "stats_dump_period_sec") {
-      new_options->stats_dump_period_sec = ParseUint32(value);
-    } else if (name == "advise_random_on_open") {
-      new_options->advise_random_on_open = ParseBoolean(name, value);
-    } else if (name == "db_write_buffer_size") {
-      new_options->db_write_buffer_size = ParseUint64(value);
-    } else if (name == "use_adaptive_mutex") {
-      new_options->use_adaptive_mutex = ParseBoolean(name, value);
-    } else if (name == "bytes_per_sync") {
-      new_options->bytes_per_sync = ParseUint64(value);
-    } else if (name == "wal_bytes_per_sync") {
-      new_options->wal_bytes_per_sync = ParseUint64(value);
     } else {
-      return false;
+      auto iter = db_options_type_info.find(name);
+      if (iter == db_options_type_info.end()) {
+        return false;
+      }
+      auto& opt_info = iter->second;
+      char* opt_address =
+          reinterpret_cast<char*>(new_options) + opt_info.offset;
+      switch (opt_info.type) {
+        case OptionType::kBoolean:
+          *reinterpret_cast<bool*>(opt_address) = ParseBoolean("", value);
+          break;
+        case OptionType::kInt:
+          *reinterpret_cast<int*>(opt_address) = ParseInt(value);
+          break;
+        case OptionType::kUInt:
+          *reinterpret_cast<unsigned int*>(opt_address) = ParseUint32(value);
+          break;
+        case OptionType::kUInt32T:
+          *reinterpret_cast<uint32_t*>(opt_address) = ParseUint32(value);
+          break;
+        case OptionType::kUInt64T:
+          *reinterpret_cast<uint64_t*>(opt_address) = ParseUint64(value);
+          break;
+        case OptionType::kSizeT:
+          *reinterpret_cast<size_t*>(opt_address) = ParseUint32(value);
+          break;
+        case OptionType::kString:
+          *reinterpret_cast<std::string*>(opt_address) = value;
+          break;
+        default:
+          return false;
+      }
     }
   } catch (const std::exception& e) {
     return false;

util/options_helper.h

@@ -17,4 +17,121 @@ Status GetMutableOptionsFromStrings(
     const std::unordered_map<std::string, std::string>& options_map,
     MutableCFOptions* new_options);
 
+enum class OptionType {
+  kBoolean,
+  kInt,
+  kUInt,
+  kUInt32T,
+  kUInt64T,
+  kSizeT,
+  kString,
+  kUnknown
+};
+
+// A struct for storing constant option information such as option name,
+// option type, and offset.
+struct OptionTypeInfo {
+  int offset;
+  OptionType type;
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
+    /*
+     // not yet supported
+      AccessHint access_hint_on_compaction_start;
+      Env* env;
+      InfoLogLevel info_log_level;
+      WALRecoveryMode wal_recovery_mode;
+      std::shared_ptr<Cache> row_cache;
+      std::shared_ptr<DeleteScheduler> delete_scheduler;
+      std::shared_ptr<Logger> info_log;
+      std::shared_ptr<RateLimiter> rate_limiter;
+      std::shared_ptr<Statistics> statistics;
+      std::vector<DbPath> db_paths;
+      std::vector<std::shared_ptr<EventListener>> listeners;
+     */
+    {"advise_random_on_open",
+     {offsetof(struct DBOptions, advise_random_on_open), OptionType::kBoolean}},
+    {"allow_mmap_reads",
+     {offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean}},
+    {"allow_mmap_writes",
+     {offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean}},
+    {"allow_os_buffer",
+     {offsetof(struct DBOptions, allow_os_buffer), OptionType::kBoolean}},
+    {"create_if_missing",
+     {offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean}},
+    {"create_missing_column_families",
+     {offsetof(struct DBOptions, create_missing_column_families),
+      OptionType::kBoolean}},
+    {"disableDataSync",
+     {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}},
+    {"disable_data_sync",  // for compatibility
+     {offsetof(struct DBOptions, disableDataSync), OptionType::kBoolean}},
+    {"enable_thread_tracking",
+     {offsetof(struct DBOptions, enable_thread_tracking),
+      OptionType::kBoolean}},
+    {"error_if_exists",
+     {offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean}},
+    {"is_fd_close_on_exec",
+     {offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean}},
+    {"paranoid_checks",
+     {offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean}},
+    {"skip_log_error_on_recovery",
+     {offsetof(struct DBOptions, skip_log_error_on_recovery),
+      OptionType::kBoolean}},
+    {"skip_stats_update_on_db_open",
+     {offsetof(struct DBOptions, skip_stats_update_on_db_open),
+      OptionType::kBoolean}},
+    {"use_adaptive_mutex",
+     {offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean}},
+    {"use_fsync",
+     {offsetof(struct DBOptions, use_fsync), OptionType::kBoolean}},
+    {"max_background_compactions",
+     {offsetof(struct DBOptions, max_background_compactions),
+      OptionType::kInt}},
+    {"max_background_flushes",
+     {offsetof(struct DBOptions, max_background_flushes), OptionType::kInt}},
+    {"max_file_opening_threads",
+     {offsetof(struct DBOptions, max_file_opening_threads), OptionType::kInt}},
+    {"max_open_files",
+     {offsetof(struct DBOptions, max_open_files), OptionType::kInt}},
+    {"table_cache_numshardbits",
+     {offsetof(struct DBOptions, table_cache_numshardbits), OptionType::kInt}},
+    {"db_write_buffer_size",
+     {offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT}},
+    {"keep_log_file_num",
+     {offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT}},
+    {"log_file_time_to_roll",
+     {offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT}},
+    {"manifest_preallocation_size",
+     {offsetof(struct DBOptions, manifest_preallocation_size),
+      OptionType::kSizeT}},
+    {"max_log_file_size",
+     {offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT}},
+    {"db_log_dir",
+     {offsetof(struct DBOptions, db_log_dir), OptionType::kString}},
+    {"wal_dir", {offsetof(struct DBOptions, wal_dir), OptionType::kString}},
+    {"num_subcompactions",
+     {offsetof(struct DBOptions, num_subcompactions), OptionType::kUInt32T}},
+    {"WAL_size_limit_MB",
+     {offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T}},
+    {"WAL_ttl_seconds",
+     {offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T}},
+    {"bytes_per_sync",
+     {offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T}},
+    {"delayed_write_rate",
+     {offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T}},
+    {"delete_obsolete_files_period_micros",
+     {offsetof(struct DBOptions, delete_obsolete_files_period_micros),
+      OptionType::kUInt64T}},
+    {"max_manifest_file_size",
+     {offsetof(struct DBOptions, max_manifest_file_size),
+      OptionType::kUInt64T}},
+    {"max_total_wal_size",
+     {offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T}},
+    {"wal_bytes_per_sync",
+     {offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T}},
+    {"stats_dump_period_sec",
+     {offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt}}};
+
 }  // namespace rocksdb

util/options_test.cc

@@ -504,6 +504,137 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) {
   ASSERT_EQ(new_options.max_open_files, 1);
   ASSERT_TRUE(new_options.rate_limiter.get() != nullptr);
 }
+
+namespace {
+void VerifyDBOptions(const DBOptions& base_opt, const DBOptions& new_opt) {
+  // boolean options
+  ASSERT_EQ(base_opt.advise_random_on_open, new_opt.advise_random_on_open);
+  ASSERT_EQ(base_opt.allow_mmap_reads, new_opt.allow_mmap_reads);
+  ASSERT_EQ(base_opt.allow_mmap_writes, new_opt.allow_mmap_writes);
+  ASSERT_EQ(base_opt.allow_os_buffer, new_opt.allow_os_buffer);
+  ASSERT_EQ(base_opt.create_if_missing, new_opt.create_if_missing);
+  ASSERT_EQ(base_opt.create_missing_column_families,
+            new_opt.create_missing_column_families);
+  ASSERT_EQ(base_opt.disableDataSync, new_opt.disableDataSync);
+  ASSERT_EQ(base_opt.enable_thread_tracking, new_opt.enable_thread_tracking);
+  ASSERT_EQ(base_opt.error_if_exists, new_opt.error_if_exists);
+  ASSERT_EQ(base_opt.is_fd_close_on_exec, new_opt.is_fd_close_on_exec);
+  ASSERT_EQ(base_opt.paranoid_checks, new_opt.paranoid_checks);
+  ASSERT_EQ(base_opt.skip_log_error_on_recovery,
+            new_opt.skip_log_error_on_recovery);
+  ASSERT_EQ(base_opt.skip_stats_update_on_db_open,
+            new_opt.skip_stats_update_on_db_open);
+  ASSERT_EQ(base_opt.use_adaptive_mutex, new_opt.use_adaptive_mutex);
+  ASSERT_EQ(base_opt.use_fsync, new_opt.use_fsync);
+
+  // int options
+  ASSERT_EQ(base_opt.max_background_compactions,
+            new_opt.max_background_compactions);
+  ASSERT_EQ(base_opt.max_background_flushes, new_opt.max_background_flushes);
+  ASSERT_EQ(base_opt.max_file_opening_threads,
+            new_opt.max_file_opening_threads);
+  ASSERT_EQ(base_opt.max_open_files, new_opt.max_open_files);
+  ASSERT_EQ(base_opt.table_cache_numshardbits,
+            new_opt.table_cache_numshardbits);
+
+  // size_t options
+  ASSERT_EQ(base_opt.db_write_buffer_size, new_opt.db_write_buffer_size);
+  ASSERT_EQ(base_opt.keep_log_file_num, new_opt.keep_log_file_num);
+  ASSERT_EQ(base_opt.log_file_time_to_roll, new_opt.log_file_time_to_roll);
+  ASSERT_EQ(base_opt.manifest_preallocation_size,
+            new_opt.manifest_preallocation_size);
+  ASSERT_EQ(base_opt.max_log_file_size, new_opt.max_log_file_size);
+
+  // std::string options
+  ASSERT_EQ(base_opt.db_log_dir, new_opt.db_log_dir);
+  ASSERT_EQ(base_opt.wal_dir, new_opt.wal_dir);
+
+  // uint32_t options
+  ASSERT_EQ(base_opt.num_subcompactions, new_opt.num_subcompactions);
+
+  // uint64_t options
+  ASSERT_EQ(base_opt.WAL_size_limit_MB, new_opt.WAL_size_limit_MB);
+  ASSERT_EQ(base_opt.WAL_ttl_seconds, new_opt.WAL_ttl_seconds);
+  ASSERT_EQ(base_opt.bytes_per_sync, new_opt.bytes_per_sync);
+  ASSERT_EQ(base_opt.delayed_write_rate, new_opt.delayed_write_rate);
+  ASSERT_EQ(base_opt.delete_obsolete_files_period_micros,
+            new_opt.delete_obsolete_files_period_micros);
+  ASSERT_EQ(base_opt.max_manifest_file_size, new_opt.max_manifest_file_size);
+  ASSERT_EQ(base_opt.max_total_wal_size, new_opt.max_total_wal_size);
+  ASSERT_EQ(base_opt.wal_bytes_per_sync, new_opt.wal_bytes_per_sync);
+
+  // unsigned int options
+  ASSERT_EQ(base_opt.stats_dump_period_sec, new_opt.stats_dump_period_sec);
+}
+}  // namespace
+
+TEST_F(OptionsTest, DBOptionsSerialization) {
+  Options base_options, new_options;
+  Random rnd(301);
+
+  // Phase 1: Make big change in base_options
+  // boolean options
+  base_options.advise_random_on_open = rnd.Uniform(2);
+  base_options.allow_mmap_reads = rnd.Uniform(2);
+  base_options.allow_mmap_writes = rnd.Uniform(2);
+  base_options.allow_os_buffer = rnd.Uniform(2);
+  base_options.create_if_missing = rnd.Uniform(2);
+  base_options.create_missing_column_families = rnd.Uniform(2);
+  base_options.disableDataSync = rnd.Uniform(2);
+  base_options.enable_thread_tracking = rnd.Uniform(2);
+  base_options.error_if_exists = rnd.Uniform(2);
+  base_options.is_fd_close_on_exec = rnd.Uniform(2);
+  base_options.paranoid_checks = rnd.Uniform(2);
+  base_options.skip_log_error_on_recovery = rnd.Uniform(2);
+  base_options.skip_stats_update_on_db_open = rnd.Uniform(2);
+  base_options.use_adaptive_mutex = rnd.Uniform(2);
+  base_options.use_fsync = rnd.Uniform(2);
+
+  // int options
+  base_options.max_background_compactions = rnd.Uniform(100);
+  base_options.max_background_flushes = rnd.Uniform(100);
+  base_options.max_file_opening_threads = rnd.Uniform(100);
+  base_options.max_open_files = rnd.Uniform(100);
+  base_options.table_cache_numshardbits = rnd.Uniform(100);
+
+  // size_t options
+  base_options.db_write_buffer_size = rnd.Uniform(10000);
+  base_options.keep_log_file_num = rnd.Uniform(10000);
+  base_options.log_file_time_to_roll = rnd.Uniform(10000);
+  base_options.manifest_preallocation_size = rnd.Uniform(10000);
+  base_options.max_log_file_size = rnd.Uniform(10000);
+
+  // std::string options
+  base_options.db_log_dir = "path/to/db_log_dir";
+  base_options.wal_dir = "path/to/wal_dir";
+
+  // uint32_t options
+  base_options.num_subcompactions = rnd.Uniform(100000);
+
+  // uint64_t options
+  static const uint64_t uint_max = static_cast<uint64_t>(UINT_MAX);
+  base_options.WAL_size_limit_MB = uint_max + rnd.Uniform(100000);
+  base_options.WAL_ttl_seconds = uint_max + rnd.Uniform(100000);
+  base_options.bytes_per_sync = uint_max + rnd.Uniform(100000);
+  base_options.delayed_write_rate = uint_max + rnd.Uniform(100000);
+  base_options.delete_obsolete_files_period_micros =
+      uint_max + rnd.Uniform(100000);
+  base_options.max_manifest_file_size = uint_max + rnd.Uniform(100000);
+  base_options.max_total_wal_size = uint_max + rnd.Uniform(100000);
+  base_options.wal_bytes_per_sync = uint_max + rnd.Uniform(100000);
+
+  // unsigned int options
+  base_options.stats_dump_period_sec = rnd.Uniform(100000);
+
+  // Phase 2: obtain a string from base_option
+  std::string base_opt_string;
+  ASSERT_OK(GetStringFromDBOptions(base_options, &base_opt_string));
+
+  // Phase 3: Set new_options from the derived string and expect
+  //          new_options == base_options
+  ASSERT_OK(GetDBOptionsFromString(DBOptions(), base_opt_string, &new_options));
+  VerifyDBOptions(base_options, new_options);
+}
 #endif  // !ROCKSDB_LITE