Bring the Configurable options together (#5753)

Summary:
This PR merges the functionality of making the ColumnFamilyOptions, TableFactory, and DBOptions into Configurable into a single PR, resolving any merge conflicts

Pull Request resolved: https://github.com/facebook/rocksdb/pull/5753

Reviewed By: ajkr

Differential Revision: D23385030

Pulled By: zhichao-cao

fbshipit-source-id: 8b977a7731556230b9b8c5a081b98e49ee4f160a
main
mrambacher 4 years ago committed by Facebook GitHub Bot
parent 18a3227b12
commit 7d472accdc
  1. 2
      .circleci/config.yml
  2. 3
      CMakeLists.txt
  3. 7
      HISTORY.md
  4. 5
      Makefile
  5. 9
      TARGETS
  6. 15
      cache/cache.cc
  7. 12
      db/column_family.cc
  8. 18
      db/db_basic_test.cc
  9. 22
      db/db_block_cache_test.cc
  10. 15
      db/db_bloom_filter_test.cc
  11. 8
      db/db_impl/db_impl_open.cc
  12. 2
      db/db_iterator_test.cc
  13. 25
      db/db_options_test.cc
  14. 4
      db/db_test.cc
  15. 10
      db/db_test2.cc
  16. 2
      db/db_test_util.h
  17. 2
      db/flush_job.cc
  18. 18
      db/internal_stats.cc
  19. 2
      db/listener_test.cc
  20. 4
      db/table_properties_collector_test.cc
  21. 5
      examples/options_file_example.cc
  22. 364
      include/rocksdb/configurable.h
  23. 6
      include/rocksdb/convenience.h
  24. 62
      include/rocksdb/table.h
  25. 12
      include/rocksdb/utilities/object_registry.h
  26. 147
      include/rocksdb/utilities/options_type.h
  27. 814
      options/cf_options.cc
  28. 2
      options/cf_options.h
  29. 610
      options/configurable.cc
  30. 211
      options/configurable_helper.h
  31. 791
      options/configurable_test.cc
  32. 127
      options/configurable_test.h
  33. 529
      options/db_options.cc
  34. 2
      options/db_options.h
  35. 2
      options/options.cc
  36. 444
      options/options_helper.cc
  37. 70
      options/options_helper.h
  38. 135
      options/options_parser.cc
  39. 8
      options/options_parser.h
  40. 3
      options/options_settable_test.cc
  41. 355
      options/options_test.cc
  42. 3
      src.mk
  43. 10
      table/adaptive/adaptive_table_factory.cc
  44. 9
      table/adaptive/adaptive_table_factory.h
  45. 247
      table/block_based/block_based_table_factory.cc
  46. 32
      table/block_based/block_based_table_factory.h
  47. 9
      table/block_based/block_based_table_reader_test.cc
  48. 2
      table/block_based/filter_policy.cc
  49. 1
      table/block_based/reader_common.h
  50. 9
      table/block_fetcher_test.cc
  51. 36
      table/cuckoo/cuckoo_table_factory.cc
  52. 22
      table/cuckoo/cuckoo_table_factory.h
  53. 8
      table/mock_table.h
  54. 95
      table/plain/plain_table_factory.cc
  55. 28
      table/plain/plain_table_factory.h
  56. 3
      table/sst_file_dumper.cc
  57. 49
      table/table_factory.cc
  58. 4
      table/table_test.cc
  59. 1
      test_util/testutil.cc
  60. 8
      tools/db_bench_tool.cc
  61. 1
      tools/trace_analyzer_tool.cc
  62. 2
      util/timer_test.cc
  63. 10
      utilities/memory/memory_test.cc
  64. 10
      utilities/options/options_util.cc
  65. 29
      utilities/options/options_util_test.cc
  66. 6
      utilities/simulator_cache/sim_cache_test.cc

@ -6,7 +6,7 @@ orbs:
executors: executors:
windows-2xlarge: windows-2xlarge:
machine: machine:
image: 'windows-server-2019-vs2019:201908-06' image: 'windows-server-2019-vs2019:stable'
resource_class: windows.2xlarge resource_class: windows.2xlarge
shell: bash.exe shell: bash.exe

@ -677,6 +677,7 @@ set(SOURCES
monitoring/thread_status_util.cc monitoring/thread_status_util.cc
monitoring/thread_status_util_debug.cc monitoring/thread_status_util_debug.cc
options/cf_options.cc options/cf_options.cc
options/configurable.cc
options/db_options.cc options/db_options.cc
options/options.cc options/options.cc
options/options_helper.cc options/options_helper.cc
@ -727,6 +728,7 @@ set(SOURCES
table/sst_file_dumper.cc table/sst_file_dumper.cc
table/sst_file_reader.cc table/sst_file_reader.cc
table/sst_file_writer.cc table/sst_file_writer.cc
table/table_factory.cc
table/table_properties.cc table/table_properties.cc
table/two_level_iterator.cc table/two_level_iterator.cc
test_util/sync_point.cc test_util/sync_point.cc
@ -1125,6 +1127,7 @@ if(WITH_TESTS)
monitoring/statistics_test.cc monitoring/statistics_test.cc
monitoring/stats_dump_scheduler_test.cc monitoring/stats_dump_scheduler_test.cc
monitoring/stats_history_test.cc monitoring/stats_history_test.cc
options/configurable_test.cc
options/options_settable_test.cc options/options_settable_test.cc
options/options_test.cc options/options_test.cc
table/block_based/block_based_filter_block_test.cc table/block_based/block_based_filter_block_test.cc

@ -43,8 +43,7 @@
* `DB::GetDbSessionId(std::string& session_id)` is added. `session_id` stores a unique identifier that gets reset every time the DB is opened. This DB session ID should be unique among all open DB instances on all hosts, and should be unique among re-openings of the same or other DBs. This identifier is recorded in the LOG file on the line starting with "DB Session ID:". * `DB::GetDbSessionId(std::string& session_id)` is added. `session_id` stores a unique identifier that gets reset every time the DB is opened. This DB session ID should be unique among all open DB instances on all hosts, and should be unique among re-openings of the same or other DBs. This identifier is recorded in the LOG file on the line starting with "DB Session ID:".
* `DB::OpenForReadOnly()` now returns `Status::NotFound` when the specified DB directory does not exist. Previously the error returned depended on the underlying `Env`. This change is available in all 6.11 releases as well. * `DB::OpenForReadOnly()` now returns `Status::NotFound` when the specified DB directory does not exist. Previously the error returned depended on the underlying `Env`. This change is available in all 6.11 releases as well.
* A parameter `verify_with_checksum` is added to `BackupEngine::VerifyBackup`, which is false by default. If it is ture, `BackupEngine::VerifyBackup` verifies checksums and file sizes of backup files. Pass `false` for `verify_with_checksum` to maintain the previous behavior and performance of `BackupEngine::VerifyBackup`, by only verifying sizes of backup files. * A parameter `verify_with_checksum` is added to `BackupEngine::VerifyBackup`, which is false by default. If it is ture, `BackupEngine::VerifyBackup` verifies checksums and file sizes of backup files. Pass `false` for `verify_with_checksum` to maintain the previous behavior and performance of `BackupEngine::VerifyBackup`, by only verifying sizes of backup files.
* Methods to configure serialize, and compare -- such as TableFactory -- are exposed directly through the Configurable base class (from which these objects inherity). This change will allow for better and more thorough configuration management and retrieval in the future
### Behavior Changes ### Behavior Changes
* Best-efforts recovery ignores CURRENT file completely. If CURRENT file is missing during recovery, best-efforts recovery still proceeds with MANIFEST file(s). * Best-efforts recovery ignores CURRENT file completely. If CURRENT file is missing during recovery, best-efforts recovery still proceeds with MANIFEST file(s).
* In best-efforts recovery, an error that is not Corruption or IOError::kNotFound or IOError::kPathNotFound will be overwritten silently. Fix this by checking all non-ok cases and return early. * In best-efforts recovery, an error that is not Corruption or IOError::kNotFound or IOError::kPathNotFound will be overwritten silently. Fix this by checking all non-ok cases and return early.
@ -68,12 +67,16 @@
* Added auto resume function to automatically recover the DB from background Retryable IO Error. When retryable IOError happens during flush and WAL write, the error is mapped to Hard Error and DB will be in read mode. When retryable IO Error happens during compaction, the error will be mapped to Soft Error. DB is still in write/read mode. Autoresume function will create a thread for a DB to call DB->ResumeImpl() to try the recover for Retryable IO Error during flush and WAL write. Compaction will be rescheduled by itself if retryable IO Error happens. Auto resume may also cause other Retryable IO Error during the recovery, so the recovery will fail. Retry the auto resume may solve the issue, so we use max_bgerror_resume_count to decide how many resume cycles will be tried in total. If it is <=0, auto resume retryable IO Error is disabled. Default is INT_MAX, which will lead to a infinit auto resume. bgerror_resume_retry_interval decides the time interval between two auto resumes. * Added auto resume function to automatically recover the DB from background Retryable IO Error. When retryable IOError happens during flush and WAL write, the error is mapped to Hard Error and DB will be in read mode. When retryable IO Error happens during compaction, the error will be mapped to Soft Error. DB is still in write/read mode. Autoresume function will create a thread for a DB to call DB->ResumeImpl() to try the recover for Retryable IO Error during flush and WAL write. Compaction will be rescheduled by itself if retryable IO Error happens. Auto resume may also cause other Retryable IO Error during the recovery, so the recovery will fail. Retry the auto resume may solve the issue, so we use max_bgerror_resume_count to decide how many resume cycles will be tried in total. If it is <=0, auto resume retryable IO Error is disabled. Default is INT_MAX, which will lead to a infinit auto resume. bgerror_resume_retry_interval decides the time interval between two auto resumes.
* Option `max_subcompactions` can be set dynamically using DB::SetDBOptions(). * Option `max_subcompactions` can be set dynamically using DB::SetDBOptions().
* Added experimental ColumnFamilyOptions::sst_partitioner_factory to define determine the partitioning of sst files. This helps compaction to split the files on interesting boundaries (key prefixes) to make propagation of sst files less write amplifying (covering the whole key space). * Added experimental ColumnFamilyOptions::sst_partitioner_factory to define determine the partitioning of sst files. This helps compaction to split the files on interesting boundaries (key prefixes) to make propagation of sst files less write amplifying (covering the whole key space).
* Methods to configure serialize, and compare -- such as TableFactory -- are exposed directly through the Configurable base class (from which these objects inherity). This change will allow for better and more thorough configuration management and retrieval in the future. The options for a Configurable object can be set via the ConfigureFromMap, ConfigureFromString, or ConfigureOption method. The serialized version of the options of an object can be retrieved via the GetOptionString, ToString, or GetOption methods. The list of options supported by an object can be obtained via the GetOptionNames method. The "raw" object (such as the BlockBasedTableOption) for an option may be retrieved via the GetOptions method. Configurable options can be compared via the AreEquivalent method. The settings within a Configurable object may be validated via the ValidateOptions method. The object may be intialized (at which point only mutable options may be updated) via the PrepareOptions method.
### Performance Improvements ### Performance Improvements
* Eliminate key copies for internal comparisons while accessing ingested block-based tables. * Eliminate key copies for internal comparisons while accessing ingested block-based tables.
* Reduce key comparisons during random access in all block-based tables. * Reduce key comparisons during random access in all block-based tables.
* BackupEngine avoids unnecessary repeated checksum computation for backing up a table file to the `shared_checksum` directory when using `kOptionalChecksumAndDbSessionId`, except on SST files generated before this version of RocksDB, which fall back on using `kChecksumAndFileSize`. * BackupEngine avoids unnecessary repeated checksum computation for backing up a table file to the `shared_checksum` directory when using `kOptionalChecksumAndDbSessionId`, except on SST files generated before this version of RocksDB, which fall back on using `kChecksumAndFileSize`.
### General Improvements
* The settings of the DBOptions and ColumnFamilyOptions are now managed by Configurable objects (see New Features). The same convenience methods to configure these options still exist but the backend implementation has been unified under a common implementation.
## 6.11 (6/12/2020) ## 6.11 (6/12/2020)
### Bug Fixes ### Bug Fixes
* Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true. * Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true.

@ -555,6 +555,7 @@ PARALLEL_TEST = \
ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1) ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
TESTS += folly_synchronization_distributed_mutex_test TESTS += folly_synchronization_distributed_mutex_test
PARALLEL_TEST += folly_synchronization_distributed_mutex_test PARALLEL_TEST += folly_synchronization_distributed_mutex_test
TESTS_PASSING_ASC = folly_synchronization_distributed_mutex_test
endif endif
# options_settable_test doesn't pass with UBSAN as we use hack in the test # options_settable_test doesn't pass with UBSAN as we use hack in the test
@ -599,6 +600,7 @@ ifdef ASSERT_STATUS_CHECKED
merger_test \ merger_test \
mock_env_test \ mock_env_test \
object_registry_test \ object_registry_test \
configurable_test \
options_settable_test \ options_settable_test \
options_test \ options_test \
random_test \ random_test \
@ -1724,6 +1726,9 @@ thread_list_test: $(OBJ_DIR)/util/thread_list_test.o $(TEST_LIBRARY) $(LIBRARY)
compact_files_test: $(OBJ_DIR)/db/compact_files_test.o $(TEST_LIBRARY) $(LIBRARY) compact_files_test: $(OBJ_DIR)/db/compact_files_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK) $(AM_LINK)
configurable_test: options/configurable_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)
options_test: $(OBJ_DIR)/options/options_test.o $(TEST_LIBRARY) $(LIBRARY) options_test: $(OBJ_DIR)/options/options_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK) $(AM_LINK)

@ -239,6 +239,7 @@ cpp_library(
"monitoring/thread_status_util.cc", "monitoring/thread_status_util.cc",
"monitoring/thread_status_util_debug.cc", "monitoring/thread_status_util_debug.cc",
"options/cf_options.cc", "options/cf_options.cc",
"options/configurable.cc",
"options/db_options.cc", "options/db_options.cc",
"options/options.cc", "options/options.cc",
"options/options_helper.cc", "options/options_helper.cc",
@ -290,6 +291,7 @@ cpp_library(
"table/sst_file_dumper.cc", "table/sst_file_dumper.cc",
"table/sst_file_reader.cc", "table/sst_file_reader.cc",
"table/sst_file_writer.cc", "table/sst_file_writer.cc",
"table/table_factory.cc",
"table/table_properties.cc", "table/table_properties.cc",
"table/two_level_iterator.cc", "table/two_level_iterator.cc",
"test_util/sync_point.cc", "test_util/sync_point.cc",
@ -716,6 +718,13 @@ ROCKS_TESTS = [
[], [],
[], [],
], ],
[
"configurable_test",
"options/configurable_test.cc",
"serial",
[],
[],
],
[ [
"corruption_test", "corruption_test",
"db/corruption_test.cc", "db/corruption_test.cc",

15
cache/cache.cc vendored

@ -10,7 +10,7 @@
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "cache/lru_cache.h" #include "cache/lru_cache.h"
#include "options/options_helper.h" #include "rocksdb/utilities/options_type.h"
#include "util/string_util.h" #include "util/string_util.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -19,22 +19,19 @@ static std::unordered_map<std::string, OptionTypeInfo>
lru_cache_options_type_info = { lru_cache_options_type_info = {
{"capacity", {"capacity",
{offsetof(struct LRUCacheOptions, capacity), OptionType::kSizeT, {offsetof(struct LRUCacheOptions, capacity), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable, OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
offsetof(struct LRUCacheOptions, capacity)}},
{"num_shard_bits", {"num_shard_bits",
{offsetof(struct LRUCacheOptions, num_shard_bits), OptionType::kInt, {offsetof(struct LRUCacheOptions, num_shard_bits), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable, OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
offsetof(struct LRUCacheOptions, num_shard_bits)}},
{"strict_capacity_limit", {"strict_capacity_limit",
{offsetof(struct LRUCacheOptions, strict_capacity_limit), {offsetof(struct LRUCacheOptions, strict_capacity_limit),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable, OptionTypeFlags::kMutable}},
offsetof(struct LRUCacheOptions, strict_capacity_limit)}},
{"high_pri_pool_ratio", {"high_pri_pool_ratio",
{offsetof(struct LRUCacheOptions, high_pri_pool_ratio), {offsetof(struct LRUCacheOptions, high_pri_pool_ratio),
OptionType::kDouble, OptionVerificationType::kNormal, OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable, OptionTypeFlags::kMutable}},
offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}}; };
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
Status Cache::CreateFromString(const ConfigOptions& config_options, Status Cache::CreateFromString(const ConfigOptions& config_options,

@ -32,7 +32,7 @@
#include "monitoring/thread_status_util.h" #include "monitoring/thread_status_util.h"
#include "options/options_helper.h" #include "options/options_helper.h"
#include "port/port.h" #include "port/port.h"
#include "table/block_based/block_based_table_factory.h" #include "rocksdb/table.h"
#include "table/merging_iterator.h" #include "table/merging_iterator.h"
#include "util/autovector.h" #include "util/autovector.h"
#include "util/cast_util.h" #include "util/cast_util.h"
@ -357,8 +357,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
result.max_compaction_bytes = result.target_file_size_base * 25; result.max_compaction_bytes = result.target_file_size_base * 25;
} }
bool is_block_based_table = bool is_block_based_table = (result.table_factory->IsInstanceOf(
(result.table_factory->Name() == BlockBasedTableFactory::kName); TableFactory::kBlockBasedTableName()));
const uint64_t kAdjustedTtl = 30 * 24 * 60 * 60; const uint64_t kAdjustedTtl = 30 * 24 * 60 * 60;
if (result.ttl == kDefaultTtl) { if (result.ttl == kDefaultTtl) {
@ -1316,7 +1316,8 @@ Status ColumnFamilyData::ValidateOptions(
} }
if (cf_options.ttl > 0 && cf_options.ttl != kDefaultTtl) { if (cf_options.ttl > 0 && cf_options.ttl != kDefaultTtl) {
if (cf_options.table_factory->Name() != BlockBasedTableFactory::kName) { if (!cf_options.table_factory->IsInstanceOf(
TableFactory::kBlockBasedTableName())) {
return Status::NotSupported( return Status::NotSupported(
"TTL is only supported in Block-Based Table format. "); "TTL is only supported in Block-Based Table format. ");
} }
@ -1324,7 +1325,8 @@ Status ColumnFamilyData::ValidateOptions(
if (cf_options.periodic_compaction_seconds > 0 && if (cf_options.periodic_compaction_seconds > 0 &&
cf_options.periodic_compaction_seconds != kDefaultPeriodicCompSecs) { cf_options.periodic_compaction_seconds != kDefaultPeriodicCompSecs) {
if (cf_options.table_factory->Name() != BlockBasedTableFactory::kName) { if (!cf_options.table_factory->IsInstanceOf(
TableFactory::kBlockBasedTableName())) {
return Status::NotSupported( return Status::NotSupported(
"Periodic Compaction is only supported in " "Periodic Compaction is only supported in "
"Block-Based Table format. "); "Block-Based Table format. ");

@ -3250,12 +3250,9 @@ TEST_P(DBBasicTestDeadline, PointLookupDeadline) {
SetTimeElapseOnlySleepOnReopen(&options); SetTimeElapseOnlySleepOnReopen(&options);
Reopen(options); Reopen(options);
if (options.table_factory && if (options.table_factory) {
!strcmp(options.table_factory->Name(), block_cache = options.table_factory->GetOptions<Cache>(
BlockBasedTableFactory::kName.c_str())) { TableFactory::kBlockCacheOpts());
BlockBasedTableFactory* bbtf =
static_cast<BlockBasedTableFactory*>(options.table_factory.get());
block_cache = bbtf->table_options().block_cache.get();
} }
Random rnd(301); Random rnd(301);
@ -3336,12 +3333,9 @@ TEST_P(DBBasicTestDeadline, IteratorDeadline) {
SetTimeElapseOnlySleepOnReopen(&options); SetTimeElapseOnlySleepOnReopen(&options);
Reopen(options); Reopen(options);
if (options.table_factory && if (options.table_factory) {
!strcmp(options.table_factory->Name(), block_cache = options.table_factory->GetOptions<Cache>(
BlockBasedTableFactory::kName.c_str())) { TableFactory::kBlockCacheOpts());
BlockBasedTableFactory* bbtf =
static_cast<BlockBasedTableFactory*>(options.table_factory.get());
block_cache = bbtf->table_options().block_cache.get();
} }
Random rnd(301); Random rnd(301);

@ -50,7 +50,7 @@ class DBBlockCacheTest : public DBTestBase {
options.avoid_flush_during_recovery = false; options.avoid_flush_during_recovery = false;
// options.compression = kNoCompression; // options.compression = kNoCompression;
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
return options; return options;
} }
@ -158,7 +158,7 @@ TEST_F(DBBlockCacheTest, IteratorBlockCacheUsage) {
std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false); std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
table_options.block_cache = cache; table_options.block_cache = cache;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options); Reopen(options);
RecordCacheCounters(options); RecordCacheCounters(options);
@ -182,7 +182,7 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false); std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
table_options.block_cache = cache; table_options.block_cache = cache;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options); Reopen(options);
RecordCacheCounters(options); RecordCacheCounters(options);
@ -238,7 +238,7 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
std::shared_ptr<Cache> compressed_cache = NewLRUCache(1 << 25, 0, false); std::shared_ptr<Cache> compressed_cache = NewLRUCache(1 << 25, 0, false);
table_options.block_cache = cache; table_options.block_cache = cache;
table_options.block_cache_compressed = compressed_cache; table_options.block_cache_compressed = compressed_cache;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options); Reopen(options);
RecordCacheCounters(options); RecordCacheCounters(options);
@ -299,7 +299,7 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksOfNewTableAddedToCache) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.cache_index_and_filter_blocks = true; table_options.cache_index_and_filter_blocks = true;
table_options.filter_policy.reset(NewBloomFilterPolicy(20)); table_options.filter_policy.reset(NewBloomFilterPolicy(20));
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
ASSERT_OK(Put(1, "key", "val")); ASSERT_OK(Put(1, "key", "val"));
@ -355,7 +355,7 @@ TEST_F(DBBlockCacheTest, FillCacheAndIterateDB) {
std::shared_ptr<Cache> cache = NewLRUCache(10, 0, true); std::shared_ptr<Cache> cache = NewLRUCache(10, 0, true);
table_options.block_cache = cache; table_options.block_cache = cache;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options); Reopen(options);
ASSERT_OK(Put("key1", "val1")); ASSERT_OK(Put("key1", "val1"));
ASSERT_OK(Put("key2", "val2")); ASSERT_OK(Put("key2", "val2"));
@ -393,7 +393,7 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksStats) {
std::shared_ptr<Cache> cache = NewLRUCache(co); std::shared_ptr<Cache> cache = NewLRUCache(co);
table_options.block_cache = cache; table_options.block_cache = cache;
table_options.filter_policy.reset(NewBloomFilterPolicy(20, true)); table_options.filter_policy.reset(NewBloomFilterPolicy(20, true));
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
ASSERT_OK(Put(1, "longer_key", "val")); ASSERT_OK(Put(1, "longer_key", "val"));
@ -474,7 +474,7 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksCachePriority) {
table_options.filter_policy.reset(NewBloomFilterPolicy(20)); table_options.filter_policy.reset(NewBloomFilterPolicy(20));
table_options.cache_index_and_filter_blocks_with_high_priority = table_options.cache_index_and_filter_blocks_with_high_priority =
priority == Cache::Priority::HIGH ? true : false; priority == Cache::Priority::HIGH ? true : false;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
DestroyAndReopen(options); DestroyAndReopen(options);
MockCache::high_pri_insert_count = 0; MockCache::high_pri_insert_count = 0;
@ -573,7 +573,7 @@ TEST_F(DBBlockCacheTest, AddRedundantStats) {
table_options.cache_index_and_filter_blocks = true; table_options.cache_index_and_filter_blocks = true;
table_options.block_cache = cache; table_options.block_cache = cache;
table_options.filter_policy.reset(NewBloomFilterPolicy(50)); table_options.filter_policy.reset(NewBloomFilterPolicy(50));
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
DestroyAndReopen(options); DestroyAndReopen(options);
// Create a new table. // Create a new table.
@ -662,7 +662,7 @@ TEST_F(DBBlockCacheTest, ParanoidFileChecks) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.cache_index_and_filter_blocks = false; table_options.cache_index_and_filter_blocks = false;
table_options.filter_policy.reset(NewBloomFilterPolicy(20)); table_options.filter_policy.reset(NewBloomFilterPolicy(20));
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);
ASSERT_OK(Put(1, "1_key", "val")); ASSERT_OK(Put(1, "1_key", "val"));
@ -846,7 +846,7 @@ TEST_F(DBBlockCacheTest, CacheCompressionDict) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.cache_index_and_filter_blocks = true; table_options.cache_index_and_filter_blocks = true;
table_options.block_cache.reset(new MockCache()); table_options.block_cache.reset(new MockCache());
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
DestroyAndReopen(options); DestroyAndReopen(options);
RecordCacheCountersForCompressionDict(options); RecordCacheCountersForCompressionDict(options);

@ -83,14 +83,17 @@ TEST_P(DBBloomFilterTestDefFormatVersion, KeyMayExist) {
options_override.partition_filters = partition_filters_; options_override.partition_filters = partition_filters_;
options_override.metadata_block_size = 32; options_override.metadata_block_size = 32;
Options options = CurrentOptions(options_override); Options options = CurrentOptions(options_override);
if (partition_filters_ && if (partition_filters_) {
static_cast<BlockBasedTableOptions*>( auto* table_options =
options.table_factory->GetOptions()) options.table_factory->GetOptions<BlockBasedTableOptions>();
->index_type != BlockBasedTableOptions::kTwoLevelIndexSearch) { if (table_options != nullptr &&
// In the current implementation partitioned filters depend on partitioned table_options->index_type !=
// indexes BlockBasedTableOptions::kTwoLevelIndexSearch) {
// In the current implementation partitioned filters depend on
// partitioned indexes
continue; continue;
} }
}
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
CreateAndReopenWithCF({"pikachu"}, options); CreateAndReopenWithCF({"pikachu"}, options);

@ -18,8 +18,8 @@
#include "monitoring/persistent_stats_history.h" #include "monitoring/persistent_stats_history.h"
#include "monitoring/stats_dump_scheduler.h" #include "monitoring/stats_dump_scheduler.h"
#include "options/options_helper.h" #include "options/options_helper.h"
#include "rocksdb/table.h"
#include "rocksdb/wal_filter.h" #include "rocksdb/wal_filter.h"
#include "table/block_based/block_based_table_factory.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "util/rate_limiter.h" #include "util/rate_limiter.h"
@ -185,12 +185,12 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
} }
namespace { namespace {
Status SanitizeOptionsByTable( Status ValidateOptionsByTable(
const DBOptions& db_opts, const DBOptions& db_opts,
const std::vector<ColumnFamilyDescriptor>& column_families) { const std::vector<ColumnFamilyDescriptor>& column_families) {
Status s; Status s;
for (auto cf : column_families) { for (auto cf : column_families) {
s = cf.options.table_factory->SanitizeOptions(db_opts, cf.options); s = ValidateOptions(db_opts, cf.options);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -1451,7 +1451,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
const std::vector<ColumnFamilyDescriptor>& column_families, const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr, std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
const bool seq_per_batch, const bool batch_per_txn) { const bool seq_per_batch, const bool batch_per_txn) {
Status s = SanitizeOptionsByTable(db_options, column_families); Status s = ValidateOptionsByTable(db_options, column_families);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }

@ -2146,7 +2146,7 @@ TEST_P(DBIteratorTest, ReadAhead) {
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.block_size = 1024; table_options.block_size = 1024;
table_options.no_block_cache = true; table_options.no_block_cache = true;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options); Reopen(options);
std::string value(1024, 'a'); std::string value(1024, 'a');

@ -33,17 +33,13 @@ class DBOptionsTest : public DBTestBase {
std::unordered_map<std::string, std::string> GetMutableDBOptionsMap( std::unordered_map<std::string, std::string> GetMutableDBOptionsMap(
const DBOptions& options) { const DBOptions& options) {
std::string options_str; std::string options_str;
std::unordered_map<std::string, std::string> mutable_map;
ConfigOptions config_options; ConfigOptions config_options;
config_options.delimiter = "; "; config_options.delimiter = "; ";
GetStringFromDBOptions(config_options, options, &options_str);
std::unordered_map<std::string, std::string> options_map; GetStringFromMutableDBOptions(config_options, MutableDBOptions(options),
StringToMap(options_str, &options_map); &options_str);
std::unordered_map<std::string, std::string> mutable_map; StringToMap(options_str, &mutable_map);
for (const auto& opt : db_options_type_info) {
if (opt.second.IsMutable() && opt.second.ShouldSerialize()) {
mutable_map[opt.first] = options_map[opt.first];
}
}
return mutable_map; return mutable_map;
} }
@ -53,15 +49,10 @@ class DBOptionsTest : public DBTestBase {
ConfigOptions config_options; ConfigOptions config_options;
config_options.delimiter = "; "; config_options.delimiter = "; ";
GetStringFromColumnFamilyOptions(config_options, options, &options_str);
std::unordered_map<std::string, std::string> options_map;
StringToMap(options_str, &options_map);
std::unordered_map<std::string, std::string> mutable_map; std::unordered_map<std::string, std::string> mutable_map;
for (const auto& opt : cf_options_type_info) { GetStringFromMutableCFOptions(config_options, MutableCFOptions(options),
if (opt.second.IsMutable() && opt.second.ShouldSerialize()) { &options_str);
mutable_map[opt.first] = options_map[opt.first]; StringToMap(options_str, &mutable_map);
}
}
return mutable_map; return mutable_map;
} }

@ -55,9 +55,7 @@
#include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/checkpoint.h"
#include "rocksdb/utilities/optimistic_transaction_db.h" #include "rocksdb/utilities/optimistic_transaction_db.h"
#include "rocksdb/utilities/write_batch_with_index.h" #include "rocksdb/utilities/write_batch_with_index.h"
#include "table/block_based/block_based_table_factory.h"
#include "table/mock_table.h" #include "table/mock_table.h"
#include "table/plain/plain_table_factory.h"
#include "table/scoped_arena_iterator.h" #include "table/scoped_arena_iterator.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "test_util/testharness.h" #include "test_util/testharness.h"
@ -3842,7 +3840,7 @@ TEST_F(DBTest, TableOptionsSanitizeTest) {
DestroyAndReopen(options); DestroyAndReopen(options);
ASSERT_EQ(db_->GetOptions().allow_mmap_reads, false); ASSERT_EQ(db_->GetOptions().allow_mmap_reads, false);
options.table_factory.reset(new PlainTableFactory()); options.table_factory.reset(NewPlainTableFactory());
options.prefix_extractor.reset(NewNoopTransform()); options.prefix_extractor.reset(NewNoopTransform());
Destroy(options); Destroy(options);
ASSERT_TRUE(!TryReopen(options).IsNotSupported()); ASSERT_TRUE(!TryReopen(options).IsNotSupported());

@ -2967,11 +2967,11 @@ TEST_F(DBTest2, OptimizeForSmallDB) {
options.OptimizeForSmallDb(); options.OptimizeForSmallDb();
// Find the cache object // Find the cache object
ASSERT_EQ(std::string(BlockBasedTableFactory::kName), ASSERT_TRUE(options.table_factory->IsInstanceOf(
std::string(options.table_factory->Name())); TableFactory::kBlockBasedTableName()));
BlockBasedTableOptions* table_options = auto table_options =
reinterpret_cast<BlockBasedTableOptions*>( options.table_factory->GetOptions<BlockBasedTableOptions>();
options.table_factory->GetOptions());
ASSERT_TRUE(table_options != nullptr); ASSERT_TRUE(table_options != nullptr);
std::shared_ptr<Cache> cache = table_options->block_cache; std::shared_ptr<Cache> cache = table_options->block_cache;

@ -38,9 +38,7 @@
#include "rocksdb/statistics.h" #include "rocksdb/statistics.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/checkpoint.h"
#include "table/block_based/block_based_table_factory.h"
#include "table/mock_table.h" #include "table/mock_table.h"
#include "table/plain/plain_table_factory.h"
#include "table/scoped_arena_iterator.h" #include "table/scoped_arena_iterator.h"
#include "test_util/mock_time_env.h" #include "test_util/mock_time_env.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"

@ -39,8 +39,6 @@
#include "rocksdb/statistics.h" #include "rocksdb/statistics.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "table/block_based/block.h"
#include "table/block_based/block_based_table_factory.h"
#include "table/merging_iterator.h" #include "table/merging_iterator.h"
#include "table/table_builder.h" #include "table/table_builder.h"
#include "table/two_level_iterator.h" #include "table/two_level_iterator.h"

@ -19,7 +19,7 @@
#include "db/column_family.h" #include "db/column_family.h"
#include "db/db_impl/db_impl.h" #include "db/db_impl/db_impl.h"
#include "table/block_based/block_based_table_factory.h" #include "rocksdb/table.h"
#include "util/string_util.h" #include "util/string_util.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -907,19 +907,9 @@ bool InternalStats::HandleBlockCacheStat(Cache** block_cache) {
assert(block_cache != nullptr); assert(block_cache != nullptr);
auto* table_factory = cfd_->ioptions()->table_factory; auto* table_factory = cfd_->ioptions()->table_factory;
assert(table_factory != nullptr); assert(table_factory != nullptr);
if (BlockBasedTableFactory::kName != table_factory->Name()) { *block_cache =
return false; table_factory->GetOptions<Cache>(TableFactory::kBlockCacheOpts());
} return *block_cache != nullptr;
auto* table_options =
reinterpret_cast<BlockBasedTableOptions*>(table_factory->GetOptions());
if (table_options == nullptr) {
return false;
}
*block_cache = table_options->block_cache.get();
if (table_options->no_block_cache || *block_cache == nullptr) {
return false;
}
return true;
} }
bool InternalStats::HandleBlockCacheCapacity(uint64_t* value, DBImpl* /*db*/, bool InternalStats::HandleBlockCacheCapacity(uint64_t* value, DBImpl* /*db*/,

@ -24,8 +24,6 @@
#include "rocksdb/slice_transform.h" #include "rocksdb/slice_transform.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "rocksdb/table_properties.h" #include "rocksdb/table_properties.h"
#include "table/block_based/block_based_table_factory.h"
#include "table/plain/plain_table_factory.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "test_util/testharness.h" #include "test_util/testharness.h"
#include "test_util/testutil.h" #include "test_util/testutil.h"

@ -3,6 +3,8 @@
// COPYING file in the root directory) and Apache 2.0 License // COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory). // (found in the LICENSE.Apache file in the root directory).
#include "db/table_properties_collector.h"
#include <map> #include <map>
#include <memory> #include <memory>
#include <string> #include <string>
@ -11,11 +13,11 @@
#include "db/db_impl/db_impl.h" #include "db/db_impl/db_impl.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/table_properties_collector.h"
#include "env/composite_env_wrapper.h" #include "env/composite_env_wrapper.h"
#include "file/sequence_file_reader.h" #include "file/sequence_file_reader.h"
#include "file/writable_file_writer.h" #include "file/writable_file_writer.h"
#include "options/cf_options.h" #include "options/cf_options.h"
#include "rocksdb/flush_block_policy.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "table/block_based/block_based_table_factory.h" #include "table/block_based/block_based_table_factory.h"
#include "table/meta_blocks.h" #include "table/meta_blocks.h"

@ -91,8 +91,9 @@ int main() {
// Initialize pointer options for each column family // Initialize pointer options for each column family
for (size_t i = 0; i < loaded_cf_descs.size(); ++i) { for (size_t i = 0; i < loaded_cf_descs.size(); ++i) {
auto* loaded_bbt_opt = reinterpret_cast<BlockBasedTableOptions*>( auto* loaded_bbt_opt =
loaded_cf_descs[0].options.table_factory->GetOptions()); loaded_cf_descs[0]
.options.table_factory->GetOptions<BlockBasedTableOptions>();
// Expect the same as BlockBasedTableOptions will be loaded form file. // Expect the same as BlockBasedTableOptions will be loaded form file.
assert(loaded_bbt_opt->block_size == bbt_opts.block_size); assert(loaded_bbt_opt->block_size == bbt_opts.block_size);
// However, block_cache needs to be manually initialized as documented // However, block_cache needs to be manually initialized as documented

@ -0,0 +1,364 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "rocksdb/rocksdb_namespace.h"
#include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE {
class Logger;
class ObjectRegistry;
class OptionTypeInfo;
struct ColumnFamilyOptions;
struct ConfigOptions;
struct DBOptions;
// Configurable is a base class used by the rocksdb that describes a
// standard way of configuring objects. A Configurable object can:
// -> Populate itself given:
// - One or more "name/value" pair strings
// - A string repesenting the set of name=value properties
// - A map of name/value properties.
// -> Convert itself into its string representation
// -> Dump itself to a Logger
// -> Compare itself to another Configurable object to see if the two objects
// have equivalent options settings
//
// If a derived class calls RegisterOptions to register (by name) how its
// options objects are to be processed, this functionality can typically be
// handled by this class without additional overrides. Otherwise, the derived
// class will need to implement the methods for handling the corresponding
// functionality.
class Configurable {
protected:
friend class ConfigurableHelper;
struct RegisteredOptions {
// The name of the options being registered
std::string name;
// Pointer to the object being registered
void* opt_ptr;
#ifndef ROCKSDB_LITE
// The map of options being registered
const std::unordered_map<std::string, OptionTypeInfo>* type_map;
#endif
};
public:
Configurable() : prepared_(false) {}
virtual ~Configurable() {}
// Returns the raw pointer of the named options that is used by this
// object, or nullptr if this function is not supported.
// Since the return value is a raw pointer, the object owns the
// pointer and the caller should not delete the pointer.
//
// Note that changing the underlying options while the object
// is currently used by any open DB is undefined behavior.
// Developers should use DB::SetOption() instead to dynamically change
// options while the DB is open.
template <typename T>
const T* GetOptions() const {
return GetOptions<T>(T::kName());
}
template <typename T>
T* GetOptions() {
return GetOptions<T>(T::kName());
}
template <typename T>
const T* GetOptions(const std::string& name) const {
return reinterpret_cast<const T*>(GetOptionsPtr(name));
}
template <typename T>
T* GetOptions(const std::string& name) {
return reinterpret_cast<T*>(const_cast<void*>(GetOptionsPtr(name)));
}
// Configures the options for this class based on the input parameters.
// On successful completion, the object is updated with the settings from
// the opt_map.
// If this method fails, an attempt is made to revert the object to original
// state. Note that the revert may not be the original state but may be an
// equivalent. For example, if the object contains an option that is a
// shared_ptr, the shared_ptr may not be the original one but a copy (e.g. not
// the Cache object that was passed in, but a Cache object of the same size).
//
// The acceptable values of the name/value pairs are documented with the
// specific class/instance.
//
// @param config_options Controls how the arguments are processed.
// @param opt_map Name/value pairs of the options to update
// @param unused If specified, this value will return the name/value
// pairs from opt_map that were NotFound for this object.
// @return OK If all values in the map were successfully updated
// If invoke_prepare_options is true, OK also implies
// PrepareOptions ran successfully.
// @return NotFound If any of the names in the opt_map were not valid
// for this object. If unused is specified, it will contain the
// collection of NotFound names.
// @return NotSupported If any of the names are valid but the object does
// not know how to convert the value. This can happen if, for example,
// there is some nested Configurable that cannot be created.
// @return InvalidArgument If any of the values cannot be successfully
// parsed. This can also be returned if PrepareOptions encounters an
// error.
// @see ConfigOptions for a description of the controls.
Status ConfigureFromMap(
const ConfigOptions& config_options,
const std::unordered_map<std::string, std::string>& opt_map);
Status ConfigureFromMap(
const ConfigOptions& config_options,
const std::unordered_map<std::string, std::string>& opt_map,
std::unordered_map<std::string, std::string>* unused);
#ifndef ROCKSDB_LITE
// Updates the named option to the input value, returning OK if successful.
// Note that ConfigureOption does not cause PrepareOptions to be invoked.
// @param config_options Controls how the name/value is processed.
// @param name The name of the option to update
// @param value The value to set for the named option
// @return OK If the named field was successfully updated to value.
// @return NotFound If the name is not valid for this object.
// @return NotSupported If the name is valid but the object does
// not know how to convert the value. This can happen if, for example,
// there is some nested Configurable that cannot be created.
// @return InvalidArgument If the value cannot be successfully parsed.
Status ConfigureOption(const ConfigOptions& config_options,
const std::string& name, const std::string& value);
#endif // ROCKSDB_LITE
// Configures the options for this class based on the input parameters.
// On successful completion, the object is updated with the settings from
// the opt_map. If this method fails, an attempt is made to revert the
// object to original state. Note that the revert may not be the original
// state but may be an equivalent.
// @see ConfigureFromMap for more details
// @param config_options Controls how the arguments are processed.
// @param opt_str string containing the values to update.
// @param unused If specified, this value will return the name/value
// pairs from opt_map that were NotFound for this object.
// @return OK If all specified values were successfully updated
// If invoke_prepare_options is true, OK also implies
// PrepareOptions ran successfully.
// @return NotFound If any of the names were not valid for this object.
// If unused is specified, it will contain the collection of NotFound
// names.
// @return NotSupported If any of the names are valid but the object does
// not know how to convert the value. This can happen if, for example,
// there is some nested Configurable that cannot be created.
// @return InvalidArgument If any of the values cannot be successfully
// parsed. This can also be returned if PrepareOptions encounters an
// error.
Status ConfigureFromString(const ConfigOptions& config_options,
const std::string& opts);
// Fills in result with the serialized options for this object.
// This is the inverse of ConfigureFromString.
// @param config_options Controls how serialization happens.
// @param result The string representation of this object.
// @return OK If the options for this object wer successfully serialized.
// @return InvalidArgument If one or more of the options could not be
// serialized.
Status GetOptionString(const ConfigOptions& config_options,
std::string* result) const;
#ifndef ROCKSDB_LITE
// Returns the serialized options for this object.
// This method is similar to GetOptionString with no errors.
// @param config_options Controls how serialization happens.
// @param prefix A string to prepend to every option.
// @return The serialized representation of the options for this object
std::string ToString(const ConfigOptions& config_options) const {
return ToString(config_options, "");
}
std::string ToString(const ConfigOptions& config_options,
const std::string& prefix) const;
// Returns the list of option names associated with this configurable
// @param config_options Controls how the names are returned
// @param result The set of option names for this object. Note that
// options that are deprecated or aliases are not returned.
// @return OK on success.
Status GetOptionNames(const ConfigOptions& config_options,
std::unordered_set<std::string>* result) const;
// Returns the value of the option associated with the input name
// This method is the functional inverse of ConfigureOption
// @param config_options Controls how the value is returned
// @param name The name of the option to return a value for.
// @param value The returned value associated with the named option.
// @return OK If the named field was successfully updated to value.
// @return NotFound If the name is not valid for this object.
// @param InvalidArgument If the name is valid for this object but
// its value cannot be serialized.
virtual Status GetOption(const ConfigOptions& config_options,
const std::string& name, std::string* value) const;
#endif // ROCKSDB_LITE
// Checks to see if this Configurable is equivalent to other.
// This method assumes that the two objects are of the same class.
// @param config_options Controls how the options are compared.
// @param other The other object to compare to.
// @param mismatch If the objects do not match, this parameter contains
// the name of the option that triggered the match failure.
// @param True if the objects match, false otherwise.
virtual bool AreEquivalent(const ConfigOptions& config_options,
const Configurable* other,
std::string* name) const;
// Returns a pretty-printed, human-readable version of the options.
// This method is typically used to dump the options to a log file.
// Classes should override this method
virtual std::string GetPrintableOptions() const { return ""; }
// Validates that the settings are valid/consistent and performs any object
// initialization required by this object. This method may be called as part
// of Configure (if invoke_prepare_options is set), or may be invoked
// separately.
//
// Once an object has been prepared, non-mutable options can no longer be
// updated.
//
// Classes must override this method to provide any implementation-specific
// initialization, such as opening log files or setting up cache parameters.
// Implementations should be idempotent (e.g. don't re-open the log file or
// reconfigure the cache), as there is the potential this method can be called
// more than once.
//
// By default, this method will also prepare all nested (Inner and
// OptionType::kConfigurable) objects.
//
// @param config_options Controls how the object is prepared. Also contains
// a Logger and Env that can be used to initialize this object.
// @return OK If the object was successfully initialized.
// @return InvalidArgument If this object could not be successfull
// initialized.
virtual Status PrepareOptions(const ConfigOptions& config_options);
// Checks to see if the settings are valid for this object.
// This method checks to see if the input DBOptions and ColumnFamilyOptions
// are valid for the settings of this object. For example, an Env might not
// support certain mmap modes or a TableFactory might require certain
// settings.
//
// By default, this method will also validate all nested (Inner and
// OptionType::kConfigurable) objects.
//
// @param db_opts The DBOptions to validate
// @param cf_opts The ColumnFamilyOptions to validate
// @return OK if the options are valid
// @return InvalidArgument If the arguments are not valid for the options
// of the current object.
virtual Status ValidateOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const;
// Returns true if this object has been initialized via PrepareOptions, false
// otherwise. Once an object has been prepared, only mutable options may be
// changed.
virtual bool IsPrepared() const { return prepared_; }
protected:
// True once the object is prepared. Once the object is prepared, only
// mutable options can be configured.
bool prepared_;
// If this class is a wrapper (has-a), this method should be
// over-written to return the inner configurable (like an EnvWrapper).
// This method should NOT recurse, but should instead return the
// direct Inner object.
virtual Configurable* Inner() const { return nullptr; }
// Returns the raw pointer for the associated named option.
// The name is typically the name of an option registered via the
// Classes may override this method to provide further specialization (such as
// returning a sub-option)
//
// The default implemntation looks at the registered options. If the
// input name matches that of a registered option, the pointer registered
// with that name is returned.
// e.g,, RegisterOptions("X", &my_ptr, ...); GetOptionsPtr("X") returns
// "my_ptr"
virtual const void* GetOptionsPtr(const std::string& name) const;
// Method for allowing options to be configured outside of the normal
// registered options framework. Classes may override this method if they
// wish to support non-standard options implementations (such as configuring
// themselves from constant or simple ":"-separated strings.
//
// The default implementation does nothing and returns OK
virtual Status ParseStringOptions(const ConfigOptions& config_options,
const std::string& opts_str);
// Internal method to configure an object from a map of name-value options.
// This method uses the input config_options to drive the configuration of
// the options in opt_map. Any option name that cannot be found from the
// input set will be returned in "unused".
//
// Classes may override this method to extend the functionality if required.
// @param config_options Controls how the options are configured and errors
// handled.
// @param opts_map The set of options to configure
// @param unused Any options from opt_map that were not configured.
// @returns a Status based on the rules outlined in ConfigureFromMap
virtual Status ConfigureOptions(
const ConfigOptions& config_options,
const std::unordered_map<std::string, std::string>& opts_map,
std::unordered_map<std::string, std::string>* unused);
#ifndef ROCKSDB_LITE
// Method that configures a the specific opt_name from opt_value.
// By default, this method calls opt_info.ParseOption with the
// input parameters.
// Classes may override this method to extend the functionality, or
// change the returned Status.
virtual Status ParseOption(const ConfigOptions& config_options,
const OptionTypeInfo& opt_info,
const std::string& opt_name,
const std::string& opt_value, void* opt_ptr);
// Internal method to see if the single option name/info matches for this and
// that Classes may override this value to change its behavior.
// @param config_options Controls how the options are being matched
// @param opt_info The OptionTypeInfo registered for this option name
// that controls what field is matched (offset) and how (type).
// @param name The name associated with this opt_info.
// @param this_ptr The base pointer to compare to. This is the object
// registered for
// for this OptionTypeInfo.
// @param that_ptr The other pointer to compare to. This is the object
// registered for
// for this OptionTypeInfo.
// @param bad_name If the match fails, the name of the option that failed to
// match.
virtual bool OptionsAreEqual(const ConfigOptions& config_options,
const OptionTypeInfo& opt_info,
const std::string& name,
const void* const this_ptr,
const void* const that_ptr,
std::string* bad_name) const;
#endif
#ifndef ROCKSDB_LITE
// Internal method to serialize options (ToString)
// Classes may override this value to change its behavior.
virtual std::string SerializeOptions(const ConfigOptions& config_options,
const std::string& header) const;
#endif // ROCKSDB_LITE
// Given a name (e.g. rocksdb.my.type.opt), returns the short name (opt)
virtual std::string GetOptionName(const std::string& long_name) const;
private:
// Contains the collection of options (name, opt_ptr, opt_map) associated with
// this object. This collection is typically set in the constructor of the
// Configurable option via
std::vector<RegisteredOptions> options_;
};
} // namespace ROCKSDB_NAMESPACE

@ -47,9 +47,15 @@ struct ConfigOptions {
// When true, any unused options will be ignored and OK will be returned // When true, any unused options will be ignored and OK will be returned
bool ignore_unknown_options = false; bool ignore_unknown_options = false;
// When true, any unsupported options will be ignored and OK will be returned
bool ignore_unsupported_options = true;
// If the strings are escaped (old-style?) // If the strings are escaped (old-style?)
bool input_strings_escaped = true; bool input_strings_escaped = true;
// Whether or not to invoke PrepareOptions after configure is called.
bool invoke_prepare_options = true;
// The separator between options when converting to a string // The separator between options when converting to a string
std::string delimiter = ";"; std::string delimiter = ";";

@ -22,15 +22,15 @@
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
#include "rocksdb/cache.h" #include "rocksdb/configurable.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/options.h" #include "rocksdb/options.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// -- Block-based Table // -- Block-based Table
class Cache;
class FilterPolicy; class FilterPolicy;
class FlushBlockPolicyFactory; class FlushBlockPolicyFactory;
class PersistentCache; class PersistentCache;
@ -53,6 +53,7 @@ enum ChecksumType : char {
// For advanced user only // For advanced user only
struct BlockBasedTableOptions { struct BlockBasedTableOptions {
static const char* kName() { return "BlockTableOptions"; };
// @flush_block_policy_factory creates the instances of flush block policy. // @flush_block_policy_factory creates the instances of flush block policy.
// which provides a configurable way to determine when to flush a block in // which provides a configurable way to determine when to flush a block in
// the block based tables. If not set, table builder will use the default // the block based tables. If not set, table builder will use the default
@ -392,6 +393,7 @@ struct PlainTablePropertyNames {
const uint32_t kPlainTableVariableLength = 0; const uint32_t kPlainTableVariableLength = 0;
struct PlainTableOptions { struct PlainTableOptions {
static const char* kName() { return "PlainTableOptions"; };
// @user_key_len: plain table has optimization for fix-sized keys, which can // @user_key_len: plain table has optimization for fix-sized keys, which can
// be specified via user_key_len. Alternatively, you can pass // be specified via user_key_len. Alternatively, you can pass
// `kPlainTableVariableLength` if your keys have variable // `kPlainTableVariableLength` if your keys have variable
@ -485,6 +487,8 @@ struct CuckooTablePropertyNames {
}; };
struct CuckooTableOptions { struct CuckooTableOptions {
static const char* kName() { return "CuckooTableOptions"; };
// Determines the utilization of hash tables. Smaller values // Determines the utilization of hash tables. Smaller values
// result in larger hash tables with fewer collisions. // result in larger hash tables with fewer collisions.
double hash_table_ratio = 0.9; double hash_table_ratio = 0.9;
@ -522,9 +526,19 @@ extern TableFactory* NewCuckooTableFactory(
class RandomAccessFileReader; class RandomAccessFileReader;
// A base class for table factories. // A base class for table factories.
class TableFactory { class TableFactory : public Configurable {
public: public:
virtual ~TableFactory() {} virtual ~TableFactory() override {}
static const char* kBlockCacheOpts() { return "BlockCache"; };
static const char* kBlockBasedTableName() { return "BlockBasedTable"; };
static const char* kPlainTableName() { return "PlainTable"; }
static const char* kCuckooTableName() { return "CuckooTable"; };
// Creates and configures a new TableFactory from the input options and id.
static Status CreateFromString(const ConfigOptions& config_options,
const std::string& id,
std::shared_ptr<TableFactory>* factory);
// The type of the table. // The type of the table.
// //
@ -535,6 +549,13 @@ class TableFactory {
// by any clients of this package. // by any clients of this package.
virtual const char* Name() const = 0; virtual const char* Name() const = 0;
// Returns true if the class is an instance of the input name.
// This is typically determined by if the input name matches the
// name of this object.
virtual bool IsInstanceOf(const std::string& name) const {
return name == Name();
}
// Returns a Table object table that can fetch data from file specified // Returns a Table object table that can fetch data from file specified
// in parameter file. It's the caller's responsibility to make sure // in parameter file. It's the caller's responsibility to make sure
// file is in the correct format. // file is in the correct format.
@ -592,39 +613,6 @@ class TableFactory {
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,
uint32_t column_family_id, WritableFileWriter* file) const = 0; uint32_t column_family_id, WritableFileWriter* file) const = 0;
// Sanitizes the specified DB Options and ColumnFamilyOptions.
//
// If the function cannot find a way to sanitize the input DB Options,
// a non-ok Status will be returned.
virtual Status SanitizeOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const = 0;
// Return a string that contains printable format of table configurations.
// RocksDB prints configurations at DB Open().
virtual std::string GetPrintableTableOptions() const = 0;
virtual Status GetOptionString(const ConfigOptions& /*config_options*/,
std::string* /*opt_string*/) const {
return Status::NotSupported(
"The table factory doesn't implement GetOptionString().");
}
// Returns the raw pointer of the table options that is used by this
// TableFactory, or nullptr if this function is not supported.
// Since the return value is a raw pointer, the TableFactory owns the
// pointer and the caller should not delete the pointer.
//
// In certain case, it is desirable to alter the underlying options when the
// TableFactory is not used by any open DB by casting the returned pointer
// to the right class. For instance, if BlockBasedTableFactory is used,
// then the pointer can be casted to BlockBasedTableOptions.
//
// Note that changing the underlying TableFactory options while the
// TableFactory is currently used by any open DB is undefined behavior.
// Developers should use DB::SetOption() instead to dynamically change
// options while the DB is open.
virtual void* GetOptions() { return nullptr; }
// Return is delete range supported // Return is delete range supported
virtual bool IsDeleteRangeSupported() const { return false; } virtual bool IsDeleteRangeSupported() const { return false; }
}; };

@ -125,7 +125,7 @@ class ObjectRegistry {
// Creates a new unique T using the input factory functions. // Creates a new unique T using the input factory functions.
// Returns OK if a new unique T was successfully created // Returns OK if a new unique T was successfully created
// Returns NotFound if the type/target could not be created // Returns NotSupported if the type/target could not be created
// Returns InvalidArgument if the factory return an unguarded object // Returns InvalidArgument if the factory return an unguarded object
// (meaning it cannot be managed by a unique ptr) // (meaning it cannot be managed by a unique ptr)
template <typename T> template <typename T>
@ -134,7 +134,7 @@ class ObjectRegistry {
std::string errmsg; std::string errmsg;
T* ptr = NewObject(target, result, &errmsg); T* ptr = NewObject(target, result, &errmsg);
if (ptr == nullptr) { if (ptr == nullptr) {
return Status::NotFound(errmsg, target); return Status::NotSupported(errmsg, target);
} else if (*result) { } else if (*result) {
return Status::OK(); return Status::OK();
} else { } else {
@ -146,7 +146,7 @@ class ObjectRegistry {
// Creates a new shared T using the input factory functions. // Creates a new shared T using the input factory functions.
// Returns OK if a new shared T was successfully created // Returns OK if a new shared T was successfully created
// Returns NotFound if the type/target could not be created // Returns NotSupported if the type/target could not be created
// Returns InvalidArgument if the factory return an unguarded object // Returns InvalidArgument if the factory return an unguarded object
// (meaning it cannot be managed by a shared ptr) // (meaning it cannot be managed by a shared ptr)
template <typename T> template <typename T>
@ -156,7 +156,7 @@ class ObjectRegistry {
std::unique_ptr<T> guard; std::unique_ptr<T> guard;
T* ptr = NewObject(target, &guard, &errmsg); T* ptr = NewObject(target, &guard, &errmsg);
if (ptr == nullptr) { if (ptr == nullptr) {
return Status::NotFound(errmsg, target); return Status::NotSupported(errmsg, target);
} else if (guard) { } else if (guard) {
result->reset(guard.release()); result->reset(guard.release());
return Status::OK(); return Status::OK();
@ -169,7 +169,7 @@ class ObjectRegistry {
// Creates a new static T using the input factory functions. // Creates a new static T using the input factory functions.
// Returns OK if a new static T was successfully created // Returns OK if a new static T was successfully created
// Returns NotFound if the type/target could not be created // Returns NotSupported if the type/target could not be created
// Returns InvalidArgument if the factory return a guarded object // Returns InvalidArgument if the factory return a guarded object
// (meaning it is managed by a unique ptr) // (meaning it is managed by a unique ptr)
template <typename T> template <typename T>
@ -178,7 +178,7 @@ class ObjectRegistry {
std::unique_ptr<T> guard; std::unique_ptr<T> guard;
T* ptr = NewObject(target, &guard, &errmsg); T* ptr = NewObject(target, &guard, &errmsg);
if (ptr == nullptr) { if (ptr == nullptr) {
return Status::NotFound(errmsg, target); return Status::NotSupported(errmsg, target);
} else if (guard.get()) { } else if (guard.get()) {
return Status::InvalidArgument(std::string("Cannot make a static ") + return Status::InvalidArgument(std::string("Cannot make a static ") +
T::Type() + " from a guarded one ", T::Type() + " from a guarded one ",

@ -16,6 +16,9 @@
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
class OptionTypeInfo; class OptionTypeInfo;
// The underlying "class/type" of the option.
// This enum is used to determine how the option should
// be converted to/from strings and compared.
enum class OptionType { enum class OptionType {
kBoolean, kBoolean,
kInt, kInt,
@ -31,7 +34,6 @@ enum class OptionType {
kCompactionPri, kCompactionPri,
kSliceTransform, kSliceTransform,
kCompressionType, kCompressionType,
kTableFactory,
kComparator, kComparator,
kCompactionFilter, kCompactionFilter,
kCompactionFilterFactory, kCompactionFilterFactory,
@ -46,6 +48,7 @@ enum class OptionType {
kEnum, kEnum,
kStruct, kStruct,
kVector, kVector,
kConfigurable,
kUnknown, kUnknown,
}; };
@ -67,6 +70,22 @@ enum class OptionVerificationType {
// independently // independently
}; };
// A set of modifier flags used to alter how an option is evaluated or
// processed. These flags can be combined together (e.g. kMutable | kShared).
// The kCompare flags can be used to control if/when options are compared.
// If kCompareNever is set, two related options would never be compared (always
// equal) If kCompareExact is set, the options will only be compared if the
// sanity mode
// is exact
// kMutable means the option can be changed after it is prepared
// kShared means the option is contained in a std::shared_ptr
// kUnique means the option is contained in a std::uniqued_ptr
// kRawPointer means the option is a raw pointer value.
// kAllowNull means that an option is allowed to be null for verification
// purposes.
// kDontSerialize means this option should not be serialized and included in
// the string representation.
// kDontPrepare means do not call PrepareOptions for this pointer value.
enum class OptionTypeFlags : uint32_t { enum class OptionTypeFlags : uint32_t {
kNone = 0x00, // No flags kNone = 0x00, // No flags
kCompareDefault = 0x0, kCompareDefault = 0x0,
@ -75,7 +94,12 @@ enum class OptionTypeFlags : uint32_t {
kCompareExact = ConfigOptions::kSanityLevelExactMatch, kCompareExact = ConfigOptions::kSanityLevelExactMatch,
kMutable = 0x0100, // Option is mutable kMutable = 0x0100, // Option is mutable
kRawPointer = 0x0200, // The option is stored as a raw pointer
kShared = 0x0400, // The option is stored as a shared_ptr
kUnique = 0x0800, // The option is stored as a unique_ptr
kAllowNull = 0x1000, // The option can be null
kDontSerialize = 0x2000, // Don't serialize the option kDontSerialize = 0x2000, // Don't serialize the option
kDontPrepare = 0x4000, // Don't prepare or sanitize this option
}; };
inline OptionTypeFlags operator|(const OptionTypeFlags &a, inline OptionTypeFlags operator|(const OptionTypeFlags &a,
@ -179,13 +203,9 @@ using EqualsFunc = std::function<bool(
// option type, and offset. // option type, and offset.
class OptionTypeInfo { class OptionTypeInfo {
public: public:
int offset_;
int mutable_offset_;
// A simple "normal", non-mutable Type "type" at offset // A simple "normal", non-mutable Type "type" at offset
OptionTypeInfo(int offset, OptionType type) OptionTypeInfo(int offset, OptionType type)
: offset_(offset), : offset_(offset),
mutable_offset_(0),
parse_func_(nullptr), parse_func_(nullptr),
serialize_func_(nullptr), serialize_func_(nullptr),
equals_func_(nullptr), equals_func_(nullptr),
@ -193,22 +213,9 @@ class OptionTypeInfo {
verification_(OptionVerificationType::kNormal), verification_(OptionVerificationType::kNormal),
flags_(OptionTypeFlags::kNone) {} flags_(OptionTypeFlags::kNone) {}
// A simple "normal", mutable Type "type" at offset
OptionTypeInfo(int offset, OptionType type, int mutable_offset)
: offset_(offset),
mutable_offset_(mutable_offset),
parse_func_(nullptr),
serialize_func_(nullptr),
equals_func_(nullptr),
type_(type),
verification_(OptionVerificationType::kNormal),
flags_(OptionTypeFlags::kMutable) {}
OptionTypeInfo(int offset, OptionType type, OptionTypeInfo(int offset, OptionType type,
OptionVerificationType verification, OptionTypeFlags flags, OptionVerificationType verification, OptionTypeFlags flags)
int mutable_offset)
: offset_(offset), : offset_(offset),
mutable_offset_(mutable_offset),
parse_func_(nullptr), parse_func_(nullptr),
serialize_func_(nullptr), serialize_func_(nullptr),
equals_func_(nullptr), equals_func_(nullptr),
@ -218,9 +225,8 @@ class OptionTypeInfo {
OptionTypeInfo(int offset, OptionType type, OptionTypeInfo(int offset, OptionType type,
OptionVerificationType verification, OptionTypeFlags flags, OptionVerificationType verification, OptionTypeFlags flags,
int mutable_offset, const ParseFunc& parse_func) const ParseFunc& parse_func)
: offset_(offset), : offset_(offset),
mutable_offset_(mutable_offset),
parse_func_(parse_func), parse_func_(parse_func),
serialize_func_(nullptr), serialize_func_(nullptr),
equals_func_(nullptr), equals_func_(nullptr),
@ -230,11 +236,10 @@ class OptionTypeInfo {
OptionTypeInfo(int offset, OptionType type, OptionTypeInfo(int offset, OptionType type,
OptionVerificationType verification, OptionTypeFlags flags, OptionVerificationType verification, OptionTypeFlags flags,
int mutable_offset, const ParseFunc& parse_func, const ParseFunc& parse_func,
const SerializeFunc& serialize_func, const SerializeFunc& serialize_func,
const EqualsFunc& equals_func) const EqualsFunc& equals_func)
: offset_(offset), : offset_(offset),
mutable_offset_(mutable_offset),
parse_func_(parse_func), parse_func_(parse_func),
serialize_func_(serialize_func), serialize_func_(serialize_func),
equals_func_(equals_func), equals_func_(equals_func),
@ -258,7 +263,7 @@ class OptionTypeInfo {
int offset, const std::unordered_map<std::string, T>* const map) { int offset, const std::unordered_map<std::string, T>* const map) {
return OptionTypeInfo( return OptionTypeInfo(
offset, OptionType::kEnum, OptionVerificationType::kNormal, offset, OptionType::kEnum, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0, OptionTypeFlags::kNone,
// Uses the map argument to convert the input string into // Uses the map argument to convert the input string into
// its corresponding enum value. If value is found in the map, // its corresponding enum value. If value is found in the map,
// addr is updated to the corresponding map entry. // addr is updated to the corresponding map entry.
@ -324,10 +329,9 @@ class OptionTypeInfo {
static OptionTypeInfo Struct( static OptionTypeInfo Struct(
const std::string& struct_name, const std::string& struct_name,
const std::unordered_map<std::string, OptionTypeInfo>* struct_map, const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
int offset, OptionVerificationType verification, OptionTypeFlags flags, int offset, OptionVerificationType verification, OptionTypeFlags flags) {
int mutable_offset) {
return OptionTypeInfo( return OptionTypeInfo(
offset, OptionType::kStruct, verification, flags, mutable_offset, offset, OptionType::kStruct, verification, flags,
// Parses the struct and updates the fields at addr // Parses the struct and updates the fields at addr
[struct_name, struct_map](const ConfigOptions& opts, [struct_name, struct_map](const ConfigOptions& opts,
const std::string& name, const std::string& name,
@ -353,10 +357,9 @@ class OptionTypeInfo {
const std::string& struct_name, const std::string& struct_name,
const std::unordered_map<std::string, OptionTypeInfo>* struct_map, const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
int offset, OptionVerificationType verification, OptionTypeFlags flags, int offset, OptionVerificationType verification, OptionTypeFlags flags,
int mutable_offset, const ParseFunc& parse_func) { const ParseFunc& parse_func) {
return OptionTypeInfo( return OptionTypeInfo(
offset, OptionType::kStruct, verification, flags, mutable_offset, offset, OptionType::kStruct, verification, flags, parse_func,
parse_func,
[struct_name, struct_map](const ConfigOptions& opts, [struct_name, struct_map](const ConfigOptions& opts,
const std::string& name, const char* addr, const std::string& name, const char* addr,
std::string* value) { std::string* value) {
@ -374,11 +377,11 @@ class OptionTypeInfo {
template <typename T> template <typename T>
static OptionTypeInfo Vector(int _offset, static OptionTypeInfo Vector(int _offset,
OptionVerificationType _verification, OptionVerificationType _verification,
OptionTypeFlags _flags, int _mutable_offset, OptionTypeFlags _flags,
const OptionTypeInfo& elem_info, const OptionTypeInfo& elem_info,
char separator = ':') { char separator = ':') {
return OptionTypeInfo( return OptionTypeInfo(
_offset, OptionType::kVector, _verification, _flags, _mutable_offset, _offset, OptionType::kVector, _verification, _flags,
[elem_info, separator](const ConfigOptions& opts, [elem_info, separator](const ConfigOptions& opts,
const std::string& name, const std::string& name,
const std::string& value, char* addr) { const std::string& value, char* addr) {
@ -450,6 +453,20 @@ class OptionTypeInfo {
} }
} }
// Returns true if the option is allowed to be null.
// Options can be null if the verification type is allow from null
// or if the flags specify allow null.
bool CanBeNull() const {
return (IsEnabled(OptionTypeFlags::kAllowNull) ||
IsEnabled(OptionVerificationType::kByNameAllowFromNull));
}
bool IsSharedPtr() const { return IsEnabled(OptionTypeFlags::kShared); }
bool IsUniquePtr() const { return IsEnabled(OptionTypeFlags::kUnique); }
bool IsRawPtr() const { return IsEnabled(OptionTypeFlags::kRawPointer); }
bool IsByName() const { bool IsByName() const {
return (verification_ == OptionVerificationType::kByName || return (verification_ == OptionVerificationType::kByName ||
verification_ == OptionVerificationType::kByNameAllowNull || verification_ == OptionVerificationType::kByNameAllowNull ||
@ -458,34 +475,82 @@ class OptionTypeInfo {
bool IsStruct() const { return (type_ == OptionType::kStruct); } bool IsStruct() const { return (type_ == OptionType::kStruct); }
bool IsConfigurable() const { return (type_ == OptionType::kConfigurable); }
// Returns the underlying pointer for the type at base_addr
// The value returned is the underlying "raw" pointer, offset from base.
template <typename T>
const T* AsRawPointer(const void* const base_addr) const {
if (base_addr == nullptr) {
return nullptr;
}
const auto opt_addr = reinterpret_cast<const char*>(base_addr) + offset_;
if (IsUniquePtr()) {
const std::unique_ptr<T>* ptr =
reinterpret_cast<const std::unique_ptr<T>*>(opt_addr);
return ptr->get();
} else if (IsSharedPtr()) {
const std::shared_ptr<T>* ptr =
reinterpret_cast<const std::shared_ptr<T>*>(opt_addr);
return ptr->get();
} else if (IsRawPtr()) {
const T* const* ptr = reinterpret_cast<const T* const*>(opt_addr);
return *ptr;
} else {
return reinterpret_cast<const T*>(opt_addr);
}
}
// Returns the underlying pointer for the type at base_addr
// The value returned is the underlying "raw" pointer, offset from base.
template <typename T>
T* AsRawPointer(void* base_addr) const {
if (base_addr == nullptr) {
return nullptr;
}
auto opt_addr = reinterpret_cast<char*>(base_addr) + offset_;
if (IsUniquePtr()) {
std::unique_ptr<T>* ptr = reinterpret_cast<std::unique_ptr<T>*>(opt_addr);
return ptr->get();
} else if (IsSharedPtr()) {
std::shared_ptr<T>* ptr = reinterpret_cast<std::shared_ptr<T>*>(opt_addr);
return ptr->get();
} else if (IsRawPtr()) {
T** ptr = reinterpret_cast<T**>(opt_addr);
return *ptr;
} else {
return reinterpret_cast<T*>(opt_addr);
}
}
// Parses the option in "opt_value" according to the rules of this class // Parses the option in "opt_value" according to the rules of this class
// and updates the value at "opt_addr". // and updates the value at "opt_ptr".
// On success, Status::OK() is returned. On failure: // On success, Status::OK() is returned. On failure:
// NotFound means the opt_name is not valid for this option // NotFound means the opt_name is not valid for this option
// NotSupported means we do not know how to parse the value for this option // NotSupported means we do not know how to parse the value for this option
// InvalidArgument means the opt_value is not valid for this option. // InvalidArgument means the opt_value is not valid for this option.
Status Parse(const ConfigOptions& config_options, const std::string& opt_name, Status Parse(const ConfigOptions& config_options, const std::string& opt_name,
const std::string& opt_value, char* opt_addr) const; const std::string& opt_value, void* const opt_ptr) const;
// Serializes the option in "opt_addr" according to the rules of this class // Serializes the option in "opt_addr" according to the rules of this class
// into the value at "opt_value". // into the value at "opt_value".
Status Serialize(const ConfigOptions& config_options, Status Serialize(const ConfigOptions& config_options,
const std::string& opt_name, const char* opt_addr, const std::string& opt_name, const void* const opt_ptr,
std::string* opt_value) const; std::string* opt_value) const;
// Compares the "addr1" and "addr2" values according to the rules of this // Compares the "addr1" and "addr2" values according to the rules of this
// class and returns true if they match. On a failed match, mismatch is the // class and returns true if they match. On a failed match, mismatch is the
// name of the option that failed to match. // name of the option that failed to match.
bool AreEqual(const ConfigOptions& config_options, bool AreEqual(const ConfigOptions& config_options,
const std::string& opt_name, const char* addr1, const std::string& opt_name, const void* const addr1,
const char* addr2, std::string* mismatch) const; const void* const addr2, std::string* mismatch) const;
// Used to override the match rules for "ByName" options. // Used to override the match rules for "ByName" options.
bool AreEqualByName(const ConfigOptions& config_options, bool AreEqualByName(const ConfigOptions& config_options,
const std::string& opt_name, const char* this_offset, const std::string& opt_name, const void* const this_ptr,
const char* that_offset) const; const void* const that_ptr) const;
bool AreEqualByName(const ConfigOptions& config_options, bool AreEqualByName(const ConfigOptions& config_options,
const std::string& opt_name, const char* this_ptr, const std::string& opt_name, const void* const this_ptr,
const std::string& that_value) const; const std::string& that_value) const;
// Parses the input value according to the map for the struct at opt_addr // Parses the input value according to the map for the struct at opt_addr
@ -553,6 +618,8 @@ class OptionTypeInfo {
size_t* end, std::string* token); size_t* end, std::string* token);
private: private:
int offset_;
// The optional function to convert a string to its representation // The optional function to convert a string to its representation
ParseFunc parse_func_; ParseFunc parse_func_;

File diff suppressed because it is too large Load Diff

@ -20,6 +20,7 @@ namespace ROCKSDB_NAMESPACE {
// of DB. Raw pointers defined in this struct do not have ownership to the data // of DB. Raw pointers defined in this struct do not have ownership to the data
// they point to. Options contains std::shared_ptr to these data. // they point to. Options contains std::shared_ptr to these data.
struct ImmutableCFOptions { struct ImmutableCFOptions {
static const char* kName() { return "ImmutableCFOptions"; }
explicit ImmutableCFOptions(const Options& options); explicit ImmutableCFOptions(const Options& options);
ImmutableCFOptions(const ImmutableDBOptions& db_options, ImmutableCFOptions(const ImmutableDBOptions& db_options,
@ -124,6 +125,7 @@ struct ImmutableCFOptions {
}; };
struct MutableCFOptions { struct MutableCFOptions {
static const char* kName() { return "MutableCFOptions"; }
explicit MutableCFOptions(const ColumnFamilyOptions& options) explicit MutableCFOptions(const ColumnFamilyOptions& options)
: write_buffer_size(options.write_buffer_size), : write_buffer_size(options.write_buffer_size),
max_write_buffer_number(options.max_write_buffer_number), max_write_buffer_number(options.max_write_buffer_number),

@ -0,0 +1,610 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "rocksdb/configurable.h"
#include "logging/logging.h"
#include "options/configurable_helper.h"
#include "options/options_helper.h"
#include "rocksdb/status.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "util/coding.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
void ConfigurableHelper::RegisterOptions(
Configurable& configurable, const std::string& name, void* opt_ptr,
const std::unordered_map<std::string, OptionTypeInfo>* type_map) {
Configurable::RegisteredOptions opts;
opts.name = name;
#ifndef ROCKSDB_LITE
opts.type_map = type_map;
#else
(void)type_map;
#endif // ROCKSDB_LITE
opts.opt_ptr = opt_ptr;
configurable.options_.emplace_back(opts);
}
//*************************************************************************
//
// Methods for Initializing and Validating Configurable Objects
//
//*************************************************************************
Status Configurable::PrepareOptions(const ConfigOptions& opts) {
Status status = Status::OK();
#ifndef ROCKSDB_LITE
for (auto opt_iter : options_) {
for (auto map_iter : *(opt_iter.type_map)) {
auto& opt_info = map_iter.second;
if (!opt_info.IsDeprecated() && !opt_info.IsAlias() &&
opt_info.IsConfigurable()) {
if (!opt_info.IsEnabled(OptionTypeFlags::kDontPrepare)) {
Configurable* config =
opt_info.AsRawPointer<Configurable>(opt_iter.opt_ptr);
if (config != nullptr) {
status = config->PrepareOptions(opts);
if (!status.ok()) {
return status;
}
}
}
}
}
}
#endif // ROCKSDB_LITE
if (status.ok()) {
auto inner = Inner();
if (inner != nullptr) {
status = inner->PrepareOptions(opts);
}
}
if (status.ok()) {
prepared_ = true;
}
return status;
}
Status Configurable::ValidateOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const {
Status status;
#ifndef ROCKSDB_LITE
for (auto opt_iter : options_) {
for (auto map_iter : *(opt_iter.type_map)) {
auto& opt_info = map_iter.second;
if (!opt_info.IsDeprecated() && !opt_info.IsAlias()) {
if (opt_info.IsConfigurable()) {
const Configurable* config =
opt_info.AsRawPointer<Configurable>(opt_iter.opt_ptr);
if (config != nullptr) {
status = config->ValidateOptions(db_opts, cf_opts);
} else if (!opt_info.CanBeNull()) {
status =
Status::NotFound("Missing configurable object", map_iter.first);
}
if (!status.ok()) {
return status;
}
}
}
}
}
#endif // ROCKSDB_LITE
if (status.ok()) {
const auto inner = Inner();
if (inner != nullptr) {
status = inner->ValidateOptions(db_opts, cf_opts);
}
}
return status;
}
/*********************************************************************************/
/* */
/* Methods for Retrieving Options from Configurables */
/* */
/*********************************************************************************/
const void* Configurable::GetOptionsPtr(const std::string& name) const {
for (auto o : options_) {
if (o.name == name) {
return o.opt_ptr;
}
}
auto inner = Inner();
if (inner != nullptr) {
return inner->GetOptionsPtr(name);
} else {
return nullptr;
}
}
std::string Configurable::GetOptionName(const std::string& opt_name) const {
return opt_name;
}
#ifndef ROCKSDB_LITE
const OptionTypeInfo* ConfigurableHelper::FindOption(
const std::vector<Configurable::RegisteredOptions>& options,
const std::string& short_name, std::string* opt_name, void** opt_ptr) {
for (auto iter : options) {
const auto opt_info =
OptionTypeInfo::Find(short_name, *(iter.type_map), opt_name);
if (opt_info != nullptr) {
*opt_ptr = iter.opt_ptr;
return opt_info;
}
}
return nullptr;
}
#endif // ROCKSDB_LITE
//*************************************************************************
//
// Methods for Configuring Options from Strings/Name-Value Pairs/Maps
//
//*************************************************************************
Status Configurable::ConfigureFromMap(
const ConfigOptions& config_options,
const std::unordered_map<std::string, std::string>& opts_map) {
Status s = ConfigureFromMap(config_options, opts_map, nullptr);
return s;
}
Status Configurable::ConfigureFromMap(
const ConfigOptions& config_options,
const std::unordered_map<std::string, std::string>& opts_map,
std::unordered_map<std::string, std::string>* unused) {
return ConfigureOptions(config_options, opts_map, unused);
}
Status Configurable::ConfigureOptions(
const ConfigOptions& config_options,
const std::unordered_map<std::string, std::string>& opts_map,
std::unordered_map<std::string, std::string>* unused) {
std::string curr_opts;
#ifndef ROCKSDB_LITE
if (!config_options.ignore_unknown_options) {
// If we are not ignoring unused, get the defaults in case we need to reset
GetOptionString(config_options, &curr_opts).PermitUncheckedError();
}
#endif // ROCKSDB_LITE
Status s = ConfigurableHelper::ConfigureOptions(config_options, *this,
opts_map, unused);
if (config_options.invoke_prepare_options && s.ok()) {
s = PrepareOptions(config_options);
}
#ifndef ROCKSDB_LITE
if (!s.ok() && !curr_opts.empty()) {
ConfigOptions reset = config_options;
reset.ignore_unknown_options = true;
reset.invoke_prepare_options = true;
// There are some options to reset from this current error
ConfigureFromString(reset, curr_opts).PermitUncheckedError();
}
#endif // ROCKSDB_LITE
return s;
}
Status Configurable::ParseStringOptions(const ConfigOptions& /*config_options*/,
const std::string& /*opts_str*/) {
return Status::OK();
}
Status Configurable::ConfigureFromString(const ConfigOptions& config_options,
const std::string& opts_str) {
Status s;
if (!opts_str.empty()) {
#ifndef ROCKSDB_LITE
if (opts_str.find(';') != std::string::npos ||
opts_str.find('=') != std::string::npos) {
std::unordered_map<std::string, std::string> opt_map;
s = StringToMap(opts_str, &opt_map);
if (s.ok()) {
s = ConfigureFromMap(config_options, opt_map, nullptr);
}
} else {
#endif // ROCKSDB_LITE
s = ParseStringOptions(config_options, opts_str);
if (s.ok() && config_options.invoke_prepare_options) {
s = PrepareOptions(config_options);
}
#ifndef ROCKSDB_LITE
}
#endif // ROCKSDB_LITE
} else if (config_options.invoke_prepare_options) {
s = PrepareOptions(config_options);
} else {
s = Status::OK();
}
return s;
}
#ifndef ROCKSDB_LITE
/**
* Sets the value of the named property to the input value, returning OK on
* succcess.
*/
Status Configurable::ConfigureOption(const ConfigOptions& config_options,
const std::string& name,
const std::string& value) {
const std::string& opt_name = GetOptionName(name);
return ConfigurableHelper::ConfigureSingleOption(config_options, *this,
opt_name, value);
}
/**
* Looks for the named option amongst the options for this type and sets
* the value for it to be the input value.
* If the name was found, found_option will be set to true and the resulting
* status should be returned.
*/
Status Configurable::ParseOption(const ConfigOptions& config_options,
const OptionTypeInfo& opt_info,
const std::string& opt_name,
const std::string& opt_value, void* opt_ptr) {
if (opt_info.IsMutable() || opt_info.IsConfigurable()) {
return opt_info.Parse(config_options, opt_name, opt_value, opt_ptr);
} else if (prepared_) {
return Status::InvalidArgument("Option not changeable: " + opt_name);
} else {
return opt_info.Parse(config_options, opt_name, opt_value, opt_ptr);
}
}
#endif // ROCKSDB_LITE
Status ConfigurableHelper::ConfigureOptions(
const ConfigOptions& config_options, Configurable& configurable,
const std::unordered_map<std::string, std::string>& opts_map,
std::unordered_map<std::string, std::string>* unused) {
std::unordered_map<std::string, std::string> remaining = opts_map;
Status s = Status::OK();
if (!opts_map.empty()) {
#ifndef ROCKSDB_LITE
for (const auto& iter : configurable.options_) {
s = ConfigureSomeOptions(config_options, configurable, *(iter.type_map),
&remaining, iter.opt_ptr);
if (remaining.empty()) { // Are there more options left?
break;
} else if (!s.ok()) {
break;
}
}
#else
(void)configurable;
if (!config_options.ignore_unknown_options) {
s = Status::NotSupported("ConfigureFromMap not supported in LITE mode");
}
#endif // ROCKSDB_LITE
}
if (unused != nullptr && !remaining.empty()) {
unused->insert(remaining.begin(), remaining.end());
}
if (config_options.ignore_unknown_options) {
s = Status::OK();
} else if (s.ok() && unused == nullptr && !remaining.empty()) {
s = Status::NotFound("Could not find option: ", remaining.begin()->first);
}
return s;
}
#ifndef ROCKSDB_LITE
/**
* Updates the object with the named-value property values, returning OK on
* succcess. Any properties that were found are removed from the options list;
* upon return only options that were not found in this opt_map remain.
* Returns:
* - OK if ignore_unknown_options is set
* - InvalidArgument, if any option was invalid
* - NotSupported, if any option is unsupported and ignore_unsupported_options
is OFF
* - OK, if no option was invalid or not supported (or ignored)
*/
Status ConfigurableHelper::ConfigureSomeOptions(
const ConfigOptions& config_options, Configurable& configurable,
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
std::unordered_map<std::string, std::string>* options, void* opt_ptr) {
Status result = Status::OK(); // The last non-OK result (if any)
Status notsup = Status::OK(); // The last NotSupported result (if any)
std::string elem_name;
int found = 1;
std::unordered_set<std::string> unsupported;
// While there are unused properties and we processed at least one,
// go through the remaining unused properties and attempt to configure them.
while (found > 0 && !options->empty()) {
found = 0;
notsup = Status::OK();
for (auto it = options->begin(); it != options->end();) {
const std::string& opt_name = configurable.GetOptionName(it->first);
const std::string& opt_value = it->second;
const auto opt_info =
OptionTypeInfo::Find(opt_name, type_map, &elem_name);
if (opt_info == nullptr) { // Did not find the option. Skip it
++it;
} else {
Status s = ConfigureOption(config_options, configurable, *opt_info,
opt_name, elem_name, opt_value, opt_ptr);
if (s.IsNotFound()) {
++it;
} else if (s.IsNotSupported()) {
notsup = s;
unsupported.insert(it->first);
++it; // Skip it for now
} else {
found++;
it = options->erase(it);
if (!s.ok()) {
result = s;
}
}
}
} // End for all remaining options
} // End while found one or options remain
// Now that we have been through the list, remove any unsupported
for (auto u : unsupported) {
auto it = options->find(u);
if (it != options->end()) {
options->erase(it);
}
}
if (config_options.ignore_unknown_options) {
if (!result.ok()) result.PermitUncheckedError();
if (!notsup.ok()) notsup.PermitUncheckedError();
return Status::OK();
} else if (!result.ok()) {
if (!notsup.ok()) notsup.PermitUncheckedError();
return result;
} else if (config_options.ignore_unsupported_options) {
if (!notsup.ok()) notsup.PermitUncheckedError();
return Status::OK();
} else {
return notsup;
}
}
Status ConfigurableHelper::ConfigureSingleOption(
const ConfigOptions& config_options, Configurable& configurable,
const std::string& name, const std::string& value) {
std::string opt_name;
void* opt_ptr = nullptr;
const auto opt_info =
FindOption(configurable.options_, name, &opt_name, &opt_ptr);
if (opt_info == nullptr) {
return Status::NotFound("Could not find option: ", name);
} else {
return ConfigureOption(config_options, configurable, *opt_info, name,
opt_name, value, opt_ptr);
}
}
Status ConfigurableHelper::ConfigureOption(
const ConfigOptions& config_options, Configurable& configurable,
const OptionTypeInfo& opt_info, const std::string& opt_name,
const std::string& name, const std::string& value, void* opt_ptr) {
if (opt_name == name) {
return configurable.ParseOption(config_options, opt_info, opt_name, value,
opt_ptr);
} else if (opt_info.IsStruct() || opt_info.IsConfigurable()) {
return configurable.ParseOption(config_options, opt_info, name, value,
opt_ptr);
} else {
return Status::NotFound("Could not find option: ", name);
}
}
#endif // ROCKSDB_LITE
//*******************************************************************************
//
// Methods for Converting Options into strings
//
//*******************************************************************************
Status Configurable::GetOptionString(const ConfigOptions& config_options,
std::string* result) const {
assert(result);
result->clear();
#ifndef ROCKSDB_LITE
return ConfigurableHelper::SerializeOptions(config_options, *this, "",
result);
#else
(void)config_options;
return Status::NotSupported("GetOptionString not supported in LITE mode");
#endif // ROCKSDB_LITE
}
#ifndef ROCKSDB_LITE
std::string Configurable::ToString(const ConfigOptions& config_options,
const std::string& prefix) const {
std::string result = SerializeOptions(config_options, prefix);
if (result.empty() || result.find('=') == std::string::npos) {
return result;
} else {
return "{" + result + "}";
}
}
std::string Configurable::SerializeOptions(const ConfigOptions& config_options,
const std::string& header) const {
std::string result;
Status s = ConfigurableHelper::SerializeOptions(config_options, *this, header,
&result);
assert(s.ok());
return result;
}
Status Configurable::GetOption(const ConfigOptions& config_options,
const std::string& name,
std::string* value) const {
return ConfigurableHelper::GetOption(config_options, *this,
GetOptionName(name), value);
}
Status ConfigurableHelper::GetOption(const ConfigOptions& config_options,
const Configurable& configurable,
const std::string& short_name,
std::string* value) {
// Look for option directly
assert(value);
value->clear();
std::string opt_name;
void* opt_ptr = nullptr;
const auto opt_info =
FindOption(configurable.options_, short_name, &opt_name, &opt_ptr);
if (opt_info != nullptr) {
ConfigOptions embedded = config_options;
embedded.delimiter = ";";
if (short_name == opt_name) {
return opt_info->Serialize(embedded, opt_name, opt_ptr, value);
} else if (opt_info->IsStruct()) {
return opt_info->Serialize(embedded, opt_name, opt_ptr, value);
} else if (opt_info->IsConfigurable()) {
auto const* config = opt_info->AsRawPointer<Configurable>(opt_ptr);
if (config != nullptr) {
return config->GetOption(embedded, opt_name, value);
}
}
}
return Status::NotFound("Cannot find option: ", short_name);
}
Status ConfigurableHelper::SerializeOptions(const ConfigOptions& config_options,
const Configurable& configurable,
const std::string& prefix,
std::string* result) {
assert(result);
for (auto const& opt_iter : configurable.options_) {
for (const auto& map_iter : *(opt_iter.type_map)) {
const auto& opt_name = map_iter.first;
const auto& opt_info = map_iter.second;
if (opt_info.ShouldSerialize()) {
std::string value;
Status s = opt_info.Serialize(config_options, prefix + opt_name,
opt_iter.opt_ptr, &value);
if (!s.ok()) {
return s;
} else if (!value.empty()) {
// <prefix><opt_name>=<value><delimiter>
result->append(prefix + opt_name + "=" + value +
config_options.delimiter);
}
}
}
}
return Status::OK();
}
#endif // ROCKSDB_LITE
//********************************************************************************
//
// Methods for listing the options from Configurables
//
//********************************************************************************
#ifndef ROCKSDB_LITE
Status Configurable::GetOptionNames(
const ConfigOptions& config_options,
std::unordered_set<std::string>* result) const {
assert(result);
return ConfigurableHelper::ListOptions(config_options, *this, "", result);
}
Status ConfigurableHelper::ListOptions(
const ConfigOptions& /*config_options*/, const Configurable& configurable,
const std::string& prefix, std::unordered_set<std::string>* result) {
Status status;
for (auto const& opt_iter : configurable.options_) {
for (const auto& map_iter : *(opt_iter.type_map)) {
const auto& opt_name = map_iter.first;
const auto& opt_info = map_iter.second;
// If the option is no longer used in rocksdb and marked as deprecated,
// we skip it in the serialization.
if (!opt_info.IsDeprecated() && !opt_info.IsAlias()) {
result->emplace(prefix + opt_name);
}
}
}
return status;
}
#endif // ROCKSDB_LITE
//*******************************************************************************
//
// Methods for Comparing Configurables
//
//*******************************************************************************
bool Configurable::AreEquivalent(const ConfigOptions& config_options,
const Configurable* other,
std::string* name) const {
assert(name);
name->clear();
if (this == other || config_options.IsCheckDisabled()) {
return true;
} else if (other != nullptr) {
#ifndef ROCKSDB_LITE
return ConfigurableHelper::AreEquivalent(config_options, *this, *other,
name);
#else
return true;
#endif // ROCKSDB_LITE
} else {
return false;
}
}
#ifndef ROCKSDB_LITE
bool Configurable::OptionsAreEqual(const ConfigOptions& config_options,
const OptionTypeInfo& opt_info,
const std::string& opt_name,
const void* const this_ptr,
const void* const that_ptr,
std::string* mismatch) const {
if (opt_info.AreEqual(config_options, opt_name, this_ptr, that_ptr,
mismatch)) {
return true;
} else if (opt_info.AreEqualByName(config_options, opt_name, this_ptr,
that_ptr)) {
*mismatch = "";
return true;
} else {
return false;
}
}
bool ConfigurableHelper::AreEquivalent(const ConfigOptions& config_options,
const Configurable& this_one,
const Configurable& that_one,
std::string* mismatch) {
assert(mismatch != nullptr);
for (auto const& o : this_one.options_) {
const auto this_offset = this_one.GetOptionsPtr(o.name);
const auto that_offset = that_one.GetOptionsPtr(o.name);
if (this_offset != that_offset) {
if (this_offset == nullptr || that_offset == nullptr) {
return false;
} else {
for (const auto& map_iter : *(o.type_map)) {
if (config_options.IsCheckEnabled(map_iter.second.GetSanityLevel()) &&
!this_one.OptionsAreEqual(config_options, map_iter.second,
map_iter.first, this_offset,
that_offset, mismatch)) {
return false;
}
}
}
}
}
return true;
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE

@ -0,0 +1,211 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include <map>
#include <stdexcept>
#include <string>
#include <vector>
#include "rocksdb/configurable.h"
#include "rocksdb/convenience.h"
namespace ROCKSDB_NAMESPACE {
// Helper class defining static methods for supporting the Configurable
// class. The purpose of this class is to keep the Configurable class
// as tight as possible and provide methods for doing the actual work
// of configuring the objects.
class ConfigurableHelper {
public:
// Registers the input name with the options and associated map.
// When classes register their options in this manner, most of the
// functionality (excluding unknown options and validate/prepare) is
// implemented by the base class.
//
// This method should be called in the class constructor to register the
// option set for this object. For example, to register the options
// associated with the BlockBasedTableFactory, the constructor calls this
// method passing in:
// - the name of the options ("BlockBasedTableOptions");
// - the options object (the BlockBasedTableOptions object for this object;
// - the options type map for the BlockBasedTableOptions.
// This registration allows the Configurable class to process the option
// values associated with the BlockBasedTableOptions without further code in
// the derived class.
//
// @param name The name of this set of options (@see GetOptionsPtr)
// @param opt_ptr Pointer to the options to associate with this name
// @param opt_map Options map that controls how this option is configured.
template <typename T>
static void RegisterOptions(
Configurable& configurable, T* opt_ptr,
const std::unordered_map<std::string, OptionTypeInfo>* opt_map) {
RegisterOptions(configurable, T::kName(), opt_ptr, opt_map);
}
static void RegisterOptions(
Configurable& configurable, const std::string& name, void* opt_ptr,
const std::unordered_map<std::string, OptionTypeInfo>* opt_map);
// Configures the input Configurable object based on the parameters.
// On successful completion, the Configurable is updated with the settings
// from the opt_map.
//
// The acceptable values of the name/value pairs are documented with the
// specific class/instance.
//
// @param config_options Controls how the arguments are processed.
// @param opt_map Name/value pairs of the options to update
// @param unused If specified, this value will return the name/value
// pairs from opt_map that were NotFound for this object.
// @return OK If all values in the map were successfully updated
// @return NotFound If any of the names in the opt_map were not valid
// for this object. If unused is specified, it will contain the
// collection of NotFound entries
// @return NotSupported If any of the names are valid but the object does
// not know how to convert the value. This can happen if, for example,
// there is some nested Configurable that cannot be created.
// @return InvalidArgument If any of the values cannot be successfully
// parsed. This can also be returned if PrepareOptions encounters an
// error.
static Status ConfigureOptions(
const ConfigOptions& config_options, Configurable& configurable,
const std::unordered_map<std::string, std::string>& options,
std::unordered_map<std::string, std::string>* unused);
#ifndef ROCKSDB_LITE
// Internal method to configure a set of options for this object.
// Classes may override this value to change its behavior.
// @param config_options Controls how the options are being configured
// @param type_name The name that was registered for this set of options
// @param type_map The map of options for this name
// @param opt_ptr Pointer to the object being configured for this option set.
// @param options The option name/values being updated. On return, any
// option that was found is removed from the list.
// @return OK If all of the options were successfully updated.
// @return InvalidArgument If an option was found but the value could not
// be updated.
// @return NotFound If an option name was not found in type_mape
// @return NotSupported If the option was found but no rule for converting
// the value could be found.
static Status ConfigureSomeOptions(
const ConfigOptions& config_options, Configurable& configurable,
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
std::unordered_map<std::string, std::string>* options, void* opt_ptr);
// Configures a single option in the input Configurable.
// This method will look through the set of option names for this
// Configurable searching for one with the input name. If such an option
// is found, it will be configured via the input value.
//
// @param config_options Controls how the option is being configured
// @param configurable The object to configure
// @param name For options with sub-options (like Structs or
// Configurables),
// this value may be the name of the sub-field of the option being
// updated. For example, if the option is
// "compaction_options_fifo.allow_compaction", then field name would be
// "allow_compaction". For most options, field_name and opt_name will be
// equivalent.
// @param value The new value for this option.
// @param See ConfigureOptions for the possible return values
static Status ConfigureSingleOption(const ConfigOptions& config_options,
Configurable& configurable,
const std::string& name,
const std::string& value);
// Configures the option referenced by opt_info for this configurable
// This method configures the option based on opt_info for the input
// configurable.
// @param config_options Controls how the option is being configured
// @param configurable The object to configure
// @param opt_name The full option name
// @param name For options with sub-options (like Structs or
// Configurables),
// this value may be the name of the sub-field of the option being
// updated. For example, if the option is
// "compaction_options_fifo.allow_compaction", then field name would be
// "allow_compaction". For most options, field_name and opt_name will be
// equivalent.
// @param value The new value for this option.
// @param See ConfigureOptions for the possible return values
static Status ConfigureOption(const ConfigOptions& config_options,
Configurable& configurable,
const OptionTypeInfo& opt_info,
const std::string& opt_name,
const std::string& name,
const std::string& value, void* opt_ptr);
// Returns the value of the option associated with the input name
// This method is the functional inverse of ConfigureOption
// @param config_options Controls how the value is returned
// @param configurable The object from which to get the option.
// @param name The name of the option to return a value for.
// @param value The returned value associated with the named option.
// Note that value will be only the serialized version
// of the option and not "name=value"
// @return OK If the named field was successfully updated to value.
// @return NotFound If the name is not valid for this object.
// @param InvalidArgument If the name is valid for this object but
// its value cannot be serialized.
static Status GetOption(const ConfigOptions& config_options,
const Configurable& configurable,
const std::string& name, std::string* value);
// Serializes the input Configurable into the output result.
// This is the inverse of ConfigureOptions
// @param config_options Controls how serialization happens.
// @param configurable The object to serialize
// @param prefix A prefix to add to the each option as it is serialized.
// @param result The string representation of the configurable.
// @return OK If the options for this object wer successfully serialized.
// @return InvalidArgument If one or more of the options could not be
// serialized.
static Status SerializeOptions(const ConfigOptions& config_options,
const Configurable& configurable,
const std::string& prefix,
std::string* result);
// Internal method to list the option names for this object.
// Classes may override this value to change its behavior.
// @see ListOptions for more details
static Status ListOptions(const ConfigOptions& config_options,
const Configurable& configurable,
const std::string& prefix,
std::unordered_set<std::string>* result);
// Checks to see if the two configurables are equivalent to one other.
// This method assumes that the two objects are of the same class.
// @param config_options Controls how the options are compared.
// @param this_one The object to compare to.
// @param that_one The other object being compared.
// @param mismatch If the objects do not match, this parameter contains
// the name of the option that triggered the match failure.
// @param True if the objects match, false otherwise.
static bool AreEquivalent(const ConfigOptions& config_options,
const Configurable& this_one,
const Configurable& that_one,
std::string* mismatch);
private:
// Looks for the option specified by name in the RegisteredOptions.
// This method traverses the types in the input options vector. If an entry
// matching name is found, that entry, opt_name, and pointer are returned.
// @param options The vector of options to search through
// @param name The name of the option to search for in the OptionType map
// @param opt_name If the name was found, this value is set to the option name
// associated with the input name/type.
// @param opt_ptr If the name was found, this value is set to the option
// pointer
// in the RegisteredOptions vector associated with this entry
// @return A pointer to the OptionTypeInfo from the options if found,
// nullptr if the name was not found in the input options
static const OptionTypeInfo* FindOption(
const std::vector<Configurable::RegisteredOptions>& options,
const std::string& name, std::string* opt_name, void** opt_ptr);
#endif // ROCKSDB_LITE
};
} // namespace ROCKSDB_NAMESPACE

@ -0,0 +1,791 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "options/configurable_test.h"
#include <cctype>
#include <cinttypes>
#include <cstring>
#include <unordered_map>
#include "options/configurable_helper.h"
#include "options/options_helper.h"
#include "options/options_parser.h"
#include "rocksdb/configurable.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
#ifndef GFLAGS
bool FLAGS_enable_print = false;
#else
#include "util/gflags_compat.h"
using GFLAGS_NAMESPACE::ParseCommandLineFlags;
DEFINE_bool(enable_print, false, "Print options generated to console.");
#endif // GFLAGS
namespace ROCKSDB_NAMESPACE {
namespace test {
class StringLogger : public Logger {
public:
using Logger::Logv;
void Logv(const char* format, va_list ap) override {
char buffer[1000];
vsnprintf(buffer, sizeof(buffer), format, ap);
string_.append(buffer);
}
const std::string& str() const { return string_; }
void clear() { string_.clear(); }
private:
std::string string_;
};
class SimpleConfigurable : public TestConfigurable<Configurable> {
public:
static SimpleConfigurable* Create(
const std::string& name = "simple",
int mode = TestConfigMode::kDefaultMode,
const std::unordered_map<std::string, OptionTypeInfo>* map =
&simple_option_info) {
return new SimpleConfigurable(name, mode, map);
}
SimpleConfigurable(const std::string& name, int mode,
const std::unordered_map<std::string, OptionTypeInfo>*
map = &simple_option_info)
: TestConfigurable(name, mode, map) {
if ((mode & TestConfigMode::kUniqueMode) != 0) {
unique_.reset(SimpleConfigurable::Create("Unique" + name_));
ConfigurableHelper::RegisterOptions(*this, name_ + "Unique", &unique_,
&unique_option_info);
}
if ((mode & TestConfigMode::kSharedMode) != 0) {
shared_.reset(SimpleConfigurable::Create("Shared" + name_));
ConfigurableHelper::RegisterOptions(*this, name_ + "Shared", &shared_,
&shared_option_info);
}
if ((mode & TestConfigMode::kRawPtrMode) != 0) {
pointer_ = SimpleConfigurable::Create("Pointer" + name_);
ConfigurableHelper::RegisterOptions(*this, name_ + "Pointer", &pointer_,
&pointer_option_info);
}
}
}; // End class SimpleConfigurable
static std::unordered_map<std::string, OptionTypeInfo> wrapped_option_info = {
#ifndef ROCKSDB_LITE
{"inner",
{0, OptionType::kConfigurable, OptionVerificationType::kNormal,
OptionTypeFlags::kShared}},
#endif // ROCKSDB_LITE
};
class WrappedConfigurable : public SimpleConfigurable {
public:
WrappedConfigurable(const std::string& name, unsigned char mode,
const std::shared_ptr<Configurable>& t)
: SimpleConfigurable(name, mode, &simple_option_info), inner_(t) {
ConfigurableHelper::RegisterOptions(*this, "WrappedOptions", &inner_,
&wrapped_option_info);
}
protected:
Configurable* Inner() const override { return inner_.get(); }
private:
std::shared_ptr<Configurable> inner_;
};
using ConfigTestFactoryFunc = std::function<Configurable*()>;
class ConfigurableTest : public testing::Test {
public:
ConfigurableTest() { config_options_.invoke_prepare_options = false; }
ConfigOptions config_options_;
};
class ConfigurableParamTest
: public ConfigurableTest,
virtual public ::testing::WithParamInterface<
std::pair<std::string, ConfigTestFactoryFunc>> {
public:
ConfigurableParamTest() {
configuration_ = GetParam().first;
factory_ = GetParam().second;
object_.reset(factory_());
}
void TestConfigureOptions(const ConfigOptions& opts);
ConfigTestFactoryFunc factory_;
std::string configuration_;
std::unique_ptr<Configurable> object_;
};
TEST_F(ConfigurableTest, GetOptionsPtrTest) {
std::string opt_str;
std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
ASSERT_NE(configurable->GetOptions<TestOptions>("simple"), nullptr);
ASSERT_EQ(configurable->GetOptions<TestOptions>("bad-opt"), nullptr);
}
TEST_F(ConfigurableTest, ConfigureFromMapTest) {
std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
auto* opts = configurable->GetOptions<TestOptions>("simple");
ASSERT_OK(configurable->ConfigureFromMap(config_options_, {}));
ASSERT_NE(opts, nullptr);
#ifndef ROCKSDB_LITE
std::unordered_map<std::string, std::string> options_map = {
{"int", "1"}, {"bool", "true"}, {"string", "string"}};
ASSERT_OK(configurable->ConfigureFromMap(config_options_, options_map));
ASSERT_EQ(opts->i, 1);
ASSERT_EQ(opts->b, true);
ASSERT_EQ(opts->s, "string");
#endif
}
TEST_F(ConfigurableTest, ConfigureFromStringTest) {
std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
auto* opts = configurable->GetOptions<TestOptions>("simple");
ASSERT_OK(configurable->ConfigureFromString(config_options_, ""));
ASSERT_NE(opts, nullptr);
#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
ASSERT_OK(configurable->ConfigureFromString(config_options_,
"int=1;bool=true;string=s"));
ASSERT_EQ(opts->i, 1);
ASSERT_EQ(opts->b, true);
ASSERT_EQ(opts->s, "s");
#endif
}
#ifndef ROCKSDB_LITE // GetOptionsFromMap is not supported in ROCKSDB_LITE
TEST_F(ConfigurableTest, ConfigureIgnoreTest) {
std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
std::unordered_map<std::string, std::string> options_map = {{"unused", "u"}};
ConfigOptions ignore = config_options_;
ignore.ignore_unknown_options = true;
ASSERT_NOK(configurable->ConfigureFromMap(config_options_, options_map));
ASSERT_OK(configurable->ConfigureFromMap(ignore, options_map));
ASSERT_NOK(configurable->ConfigureFromString(config_options_, "unused=u"));
ASSERT_OK(configurable->ConfigureFromString(ignore, "unused=u"));
}
TEST_F(ConfigurableTest, ConfigureNestedOptionsTest) {
std::unique_ptr<Configurable> base, copy;
std::string opt_str;
std::string mismatch;
base.reset(SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode));
copy.reset(SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode));
ASSERT_OK(base->ConfigureFromString(config_options_,
"shared={int=10; string=10};"
"unique={int=20; string=20};"
"pointer={int=30; string=30};"));
ASSERT_OK(base->GetOptionString(config_options_, &opt_str));
ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
}
TEST_F(ConfigurableTest, GetOptionsTest) {
std::unique_ptr<Configurable> simple;
simple.reset(
SimpleConfigurable::Create("simple", TestConfigMode::kAllOptMode));
int i = 11;
for (auto opt : {"", "shared.", "unique.", "pointer."}) {
std::string value;
std::string expected = ToString(i);
std::string opt_name = opt;
ASSERT_OK(
simple->ConfigureOption(config_options_, opt_name + "int", expected));
ASSERT_OK(simple->GetOption(config_options_, opt_name + "int", &value));
ASSERT_EQ(expected, value);
ASSERT_OK(simple->ConfigureOption(config_options_, opt_name + "string",
expected));
ASSERT_OK(simple->GetOption(config_options_, opt_name + "string", &value));
ASSERT_EQ(expected, value);
ASSERT_NOK(
simple->ConfigureOption(config_options_, opt_name + "bad", expected));
ASSERT_NOK(simple->GetOption(config_options_, "bad option", &value));
ASSERT_TRUE(value.empty());
i += 11;
}
}
TEST_F(ConfigurableTest, ConfigureBadOptionsTest) {
std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
auto* opts = configurable->GetOptions<TestOptions>("simple");
ASSERT_NE(opts, nullptr);
ASSERT_OK(configurable->ConfigureOption(config_options_, "int", "42"));
ASSERT_EQ(opts->i, 42);
ASSERT_NOK(configurable->ConfigureOption(config_options_, "int", "fred"));
ASSERT_NOK(configurable->ConfigureOption(config_options_, "bool", "fred"));
ASSERT_NOK(
configurable->ConfigureFromString(config_options_, "int=33;unused=u"));
ASSERT_EQ(opts->i, 42);
}
TEST_F(ConfigurableTest, InvalidOptionTest) {
std::unique_ptr<Configurable> configurable(SimpleConfigurable::Create());
std::unordered_map<std::string, std::string> options_map = {
{"bad-option", "bad"}};
ASSERT_NOK(configurable->ConfigureFromMap(config_options_, options_map));
ASSERT_NOK(
configurable->ConfigureFromString(config_options_, "bad-option=bad"));
ASSERT_NOK(
configurable->ConfigureOption(config_options_, "bad-option", "bad"));
}
static std::unordered_map<std::string, OptionTypeInfo> validated_option_info = {
#ifndef ROCKSDB_LITE
{"validated",
{0, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
#endif // ROCKSDB_LITE
};
static std::unordered_map<std::string, OptionTypeInfo> prepared_option_info = {
#ifndef ROCKSDB_LITE
{"prepared",
{0, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
#endif // ROCKSDB_LITE
};
static std::unordered_map<std::string, OptionTypeInfo>
dont_prepare_option_info = {
#ifndef ROCKSDB_LITE
{"unique",
{0, OptionType::kConfigurable, OptionVerificationType::kNormal,
(OptionTypeFlags::kUnique | OptionTypeFlags::kDontPrepare)}},
#endif // ROCKSDB_LITE
};
class ValidatedConfigurable : public SimpleConfigurable {
public:
ValidatedConfigurable(const std::string& name, unsigned char mode,
bool dont_prepare = false)
: SimpleConfigurable(name, TestConfigMode::kDefaultMode),
validated(false),
prepared(0) {
ConfigurableHelper::RegisterOptions(*this, "Validated", &validated,
&validated_option_info);
ConfigurableHelper::RegisterOptions(*this, "Prepared", &prepared,
&prepared_option_info);
if ((mode & TestConfigMode::kUniqueMode) != 0) {
unique_.reset(new ValidatedConfigurable(
"Unique" + name_, TestConfigMode::kDefaultMode, false));
if (dont_prepare) {
ConfigurableHelper::RegisterOptions(*this, name_ + "Unique", &unique_,
&dont_prepare_option_info);
} else {
ConfigurableHelper::RegisterOptions(*this, name_ + "Unique", &unique_,
&unique_option_info);
}
}
}
Status PrepareOptions(const ConfigOptions& config_options) override {
if (++prepared <= 0) {
return Status::InvalidArgument("Cannot prepare option");
} else {
return SimpleConfigurable::PrepareOptions(config_options);
}
}
Status ValidateOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const override {
if (!validated) {
return Status::InvalidArgument("Not Validated");
} else {
return SimpleConfigurable::ValidateOptions(db_opts, cf_opts);
}
}
private:
bool validated;
int prepared;
};
TEST_F(ConfigurableTest, ValidateOptionsTest) {
std::unique_ptr<Configurable> configurable(
new ValidatedConfigurable("validated", TestConfigMode::kDefaultMode));
ColumnFamilyOptions cf_opts;
DBOptions db_opts;
ASSERT_OK(
configurable->ConfigureOption(config_options_, "validated", "false"));
ASSERT_NOK(configurable->ValidateOptions(db_opts, cf_opts));
ASSERT_OK(
configurable->ConfigureOption(config_options_, "validated", "true"));
ASSERT_OK(configurable->ValidateOptions(db_opts, cf_opts));
}
TEST_F(ConfigurableTest, PrepareOptionsTest) {
std::unique_ptr<Configurable> c(
new ValidatedConfigurable("Simple", TestConfigMode::kUniqueMode, false));
auto cp = c->GetOptions<int>("Prepared");
auto u = c->GetOptions<std::unique_ptr<Configurable>>("SimpleUnique");
auto up = u->get()->GetOptions<int>("Prepared");
config_options_.invoke_prepare_options = false;
ASSERT_NE(cp, nullptr);
ASSERT_NE(up, nullptr);
ASSERT_EQ(*cp, 0);
ASSERT_EQ(*up, 0);
ASSERT_OK(c->ConfigureFromMap(config_options_, {}));
ASSERT_EQ(*cp, 0);
ASSERT_EQ(*up, 0);
config_options_.invoke_prepare_options = true;
ASSERT_OK(c->ConfigureFromMap(config_options_, {}));
ASSERT_EQ(*cp, 1);
ASSERT_EQ(*up, 1);
ASSERT_OK(c->ConfigureFromString(config_options_, "prepared=0"));
ASSERT_EQ(*up, 2);
ASSERT_EQ(*cp, 1);
ASSERT_NOK(c->ConfigureFromString(config_options_, "prepared=-2"));
c.reset(
new ValidatedConfigurable("Simple", TestConfigMode::kUniqueMode, true));
cp = c->GetOptions<int>("Prepared");
u = c->GetOptions<std::unique_ptr<Configurable>>("SimpleUnique");
up = u->get()->GetOptions<int>("Prepared");
ASSERT_OK(c->ConfigureFromString(config_options_, "prepared=0"));
ASSERT_EQ(*cp, 1);
ASSERT_EQ(*up, 0);
}
TEST_F(ConfigurableTest, DeprecatedOptionsTest) {
static std::unordered_map<std::string, OptionTypeInfo>
deprecated_option_info = {
{"deprecated",
{offsetof(struct TestOptions, b), OptionType::kBoolean,
OptionVerificationType::kDeprecated, OptionTypeFlags::kNone}}};
std::unique_ptr<Configurable> orig;
orig.reset(SimpleConfigurable::Create("simple", TestConfigMode::kDefaultMode,
&deprecated_option_info));
auto* opts = orig->GetOptions<TestOptions>("simple");
ASSERT_NE(opts, nullptr);
opts->d = true;
ASSERT_OK(orig->ConfigureOption(config_options_, "deprecated", "false"));
ASSERT_TRUE(opts->d);
ASSERT_OK(orig->ConfigureFromString(config_options_, "deprecated=false"));
ASSERT_TRUE(opts->d);
}
TEST_F(ConfigurableTest, AliasOptionsTest) {
static std::unordered_map<std::string, OptionTypeInfo> alias_option_info = {
{"bool",
{offsetof(struct TestOptions, b), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"alias",
{offsetof(struct TestOptions, b), OptionType::kBoolean,
OptionVerificationType::kAlias, OptionTypeFlags::kNone, 0}}};
std::unique_ptr<Configurable> orig;
orig.reset(SimpleConfigurable::Create("simple", TestConfigMode::kDefaultMode,
&alias_option_info));
auto* opts = orig->GetOptions<TestOptions>("simple");
ASSERT_NE(opts, nullptr);
ASSERT_OK(orig->ConfigureOption(config_options_, "bool", "false"));
ASSERT_FALSE(opts->b);
ASSERT_OK(orig->ConfigureOption(config_options_, "alias", "true"));
ASSERT_TRUE(opts->b);
std::string opts_str;
ASSERT_OK(orig->GetOptionString(config_options_, &opts_str));
ASSERT_EQ(opts_str.find("alias"), std::string::npos);
ASSERT_OK(orig->ConfigureOption(config_options_, "bool", "false"));
ASSERT_FALSE(opts->b);
ASSERT_OK(orig->GetOption(config_options_, "alias", &opts_str));
ASSERT_EQ(opts_str, "false");
}
TEST_F(ConfigurableTest, NestedUniqueConfigTest) {
std::unique_ptr<Configurable> simple;
simple.reset(
SimpleConfigurable::Create("Outer", TestConfigMode::kAllOptMode));
const auto outer = simple->GetOptions<TestOptions>("Outer");
const auto unique =
simple->GetOptions<std::unique_ptr<Configurable>>("OuterUnique");
ASSERT_NE(outer, nullptr);
ASSERT_NE(unique, nullptr);
ASSERT_OK(
simple->ConfigureFromString(config_options_, "int=24;string=outer"));
ASSERT_OK(simple->ConfigureFromString(config_options_,
"unique={int=42;string=nested}"));
const auto inner = unique->get()->GetOptions<TestOptions>("UniqueOuter");
ASSERT_NE(inner, nullptr);
ASSERT_EQ(outer->i, 24);
ASSERT_EQ(outer->s, "outer");
ASSERT_EQ(inner->i, 42);
ASSERT_EQ(inner->s, "nested");
}
TEST_F(ConfigurableTest, NestedSharedConfigTest) {
std::unique_ptr<Configurable> simple;
simple.reset(SimpleConfigurable::Create(
"Outer", TestConfigMode::kDefaultMode | TestConfigMode::kSharedMode));
ASSERT_OK(
simple->ConfigureFromString(config_options_, "int=24;string=outer"));
ASSERT_OK(simple->ConfigureFromString(config_options_,
"shared={int=42;string=nested}"));
const auto outer = simple->GetOptions<TestOptions>("Outer");
const auto shared =
simple->GetOptions<std::shared_ptr<Configurable>>("OuterShared");
ASSERT_NE(outer, nullptr);
ASSERT_NE(shared, nullptr);
const auto inner = shared->get()->GetOptions<TestOptions>("SharedOuter");
ASSERT_NE(inner, nullptr);
ASSERT_EQ(outer->i, 24);
ASSERT_EQ(outer->s, "outer");
ASSERT_EQ(inner->i, 42);
ASSERT_EQ(inner->s, "nested");
}
TEST_F(ConfigurableTest, NestedRawConfigTest) {
std::unique_ptr<Configurable> simple;
simple.reset(SimpleConfigurable::Create(
"Outer", TestConfigMode::kDefaultMode | TestConfigMode::kRawPtrMode));
ASSERT_OK(
simple->ConfigureFromString(config_options_, "int=24;string=outer"));
ASSERT_OK(simple->ConfigureFromString(config_options_,
"pointer={int=42;string=nested}"));
const auto outer = simple->GetOptions<TestOptions>("Outer");
const auto pointer = simple->GetOptions<Configurable*>("OuterPointer");
ASSERT_NE(outer, nullptr);
ASSERT_NE(pointer, nullptr);
const auto inner = (*pointer)->GetOptions<TestOptions>("PointerOuter");
ASSERT_NE(inner, nullptr);
ASSERT_EQ(outer->i, 24);
ASSERT_EQ(outer->s, "outer");
ASSERT_EQ(inner->i, 42);
ASSERT_EQ(inner->s, "nested");
}
TEST_F(ConfigurableTest, MatchesTest) {
std::string mismatch;
std::unique_ptr<Configurable> base, copy;
base.reset(SimpleConfigurable::Create(
"simple", TestConfigMode::kDefaultMode | TestConfigMode::kNestedMode));
copy.reset(SimpleConfigurable::Create(
"simple", TestConfigMode::kDefaultMode | TestConfigMode::kNestedMode));
ASSERT_OK(base->ConfigureFromString(
config_options_,
"int=11;string=outer;unique={int=22;string=u};shared={int=33;string=s}"));
ASSERT_OK(copy->ConfigureFromString(
config_options_,
"int=11;string=outer;unique={int=22;string=u};shared={int=33;string=s}"));
ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_OK(base->ConfigureOption(config_options_, "shared", "int=44"));
ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_EQ(mismatch, "shared.int");
std::string c1value, c2value;
ASSERT_OK(base->GetOption(config_options_, mismatch, &c1value));
ASSERT_OK(copy->GetOption(config_options_, mismatch, &c2value));
ASSERT_NE(c1value, c2value);
}
static Configurable* SimpleStructFactory() {
static std::unordered_map<std::string, OptionTypeInfo> struct_option_info = {
#ifndef ROCKSDB_LITE
{"struct", OptionTypeInfo::Struct("struct", &simple_option_info, 0,
OptionVerificationType::kNormal,
OptionTypeFlags::kMutable)},
#endif // ROCKSDB_LITE
};
return SimpleConfigurable::Create(
"simple-struct", TestConfigMode::kDefaultMode, &struct_option_info);
}
TEST_F(ConfigurableTest, ConfigureStructTest) {
std::unique_ptr<Configurable> base(SimpleStructFactory());
std::unique_ptr<Configurable> copy(SimpleStructFactory());
std::string opt_str, value;
std::string mismatch;
std::unordered_set<std::string> names;
ASSERT_OK(
base->ConfigureFromString(config_options_, "struct={int=10; string=10}"));
ASSERT_OK(base->GetOptionString(config_options_, &opt_str));
ASSERT_OK(copy->ConfigureFromString(config_options_, opt_str));
ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_OK(base->GetOptionNames(config_options_, &names));
ASSERT_EQ(names.size(), 1);
ASSERT_EQ(*(names.begin()), "struct");
ASSERT_OK(
base->ConfigureFromString(config_options_, "struct={int=20; string=20}"));
ASSERT_OK(base->GetOption(config_options_, "struct", &value));
ASSERT_OK(copy->ConfigureOption(config_options_, "struct", value));
ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_NOK(base->ConfigureFromString(config_options_,
"struct={int=10; string=10; bad=11}"));
ASSERT_OK(base->ConfigureOption(config_options_, "struct.int", "42"));
ASSERT_NOK(base->ConfigureOption(config_options_, "struct.bad", "42"));
ASSERT_NOK(base->GetOption(config_options_, "struct.bad", &value));
ASSERT_OK(base->GetOption(config_options_, "struct.int", &value));
ASSERT_EQ(value, "42");
}
TEST_F(ConfigurableTest, ConfigurableEnumTest) {
std::unique_ptr<Configurable> base, copy;
base.reset(SimpleConfigurable::Create("e", TestConfigMode::kEnumMode));
copy.reset(SimpleConfigurable::Create("e", TestConfigMode::kEnumMode));
std::string opts_str;
std::string mismatch;
ASSERT_OK(base->ConfigureFromString(config_options_, "enum=B"));
ASSERT_FALSE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_OK(base->GetOptionString(config_options_, &opts_str));
ASSERT_OK(copy->ConfigureFromString(config_options_, opts_str));
ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_NOK(base->ConfigureOption(config_options_, "enum", "bad"));
ASSERT_NOK(base->ConfigureOption(config_options_, "unknown", "bad"));
}
#ifndef ROCKSDB_LITE
static std::unordered_map<std::string, OptionTypeInfo> noserialize_option_info =
{
{"int",
{offsetof(struct TestOptions, i), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kDontSerialize}},
};
TEST_F(ConfigurableTest, TestNoSerialize) {
std::unique_ptr<Configurable> base;
base.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode,
&noserialize_option_info));
std::string opts_str, value;
ASSERT_OK(base->ConfigureFromString(config_options_, "int=10"));
ASSERT_OK(base->GetOptionString(config_options_, &opts_str));
ASSERT_EQ(opts_str, "");
ASSERT_NOK(base->GetOption(config_options_, "int", &value));
}
TEST_F(ConfigurableTest, TestNoCompare) {
std::unordered_map<std::string, OptionTypeInfo> nocomp_option_info = {
{"int",
{offsetof(struct TestOptions, i), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kCompareNever}},
};
std::unordered_map<std::string, OptionTypeInfo> normal_option_info = {
{"int",
{offsetof(struct TestOptions, i), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
};
std::unique_ptr<Configurable> base, copy;
base.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode,
&nocomp_option_info));
copy.reset(SimpleConfigurable::Create("c", TestConfigMode::kDefaultMode,
&normal_option_info));
ASSERT_OK(base->ConfigureFromString(config_options_, "int=10"));
ASSERT_OK(copy->ConfigureFromString(config_options_, "int=20"));
std::string bvalue, cvalue, mismatch;
ASSERT_OK(base->GetOption(config_options_, "int", &bvalue));
ASSERT_OK(copy->GetOption(config_options_, "int", &cvalue));
ASSERT_EQ(bvalue, "10");
ASSERT_EQ(cvalue, "20");
ASSERT_TRUE(base->AreEquivalent(config_options_, copy.get(), &mismatch));
ASSERT_FALSE(copy->AreEquivalent(config_options_, base.get(), &mismatch));
}
#endif
void ConfigurableParamTest::TestConfigureOptions(
const ConfigOptions& config_options) {
std::unique_ptr<Configurable> base, copy;
std::unordered_set<std::string> names;
std::string opt_str, mismatch;
base.reset(factory_());
copy.reset(factory_());
ASSERT_OK(base->ConfigureFromString(config_options, configuration_));
ASSERT_OK(base->GetOptionString(config_options, &opt_str));
ASSERT_OK(copy->ConfigureFromString(config_options, opt_str));
ASSERT_OK(copy->GetOptionString(config_options, &opt_str));
ASSERT_TRUE(base->AreEquivalent(config_options, copy.get(), &mismatch));
copy.reset(factory_());
ASSERT_OK(base->GetOptionNames(config_options, &names));
std::unordered_map<std::string, std::string> unused;
bool found_one = false;
for (auto name : names) {
std::string value;
Status s = base->GetOption(config_options, name, &value);
if (s.ok()) {
s = copy->ConfigureOption(config_options, name, value);
if (s.ok() || s.IsNotSupported()) {
found_one = true;
} else {
unused[name] = value;
}
} else {
ASSERT_TRUE(s.IsNotSupported());
}
}
ASSERT_TRUE(found_one || names.empty());
while (found_one && !unused.empty()) {
found_one = false;
for (auto iter = unused.begin(); iter != unused.end();) {
if (copy->ConfigureOption(config_options, iter->first, iter->second)
.ok()) {
found_one = true;
iter = unused.erase(iter);
} else {
++iter;
}
}
}
ASSERT_EQ(0, unused.size());
ASSERT_TRUE(base->AreEquivalent(config_options, copy.get(), &mismatch));
}
TEST_P(ConfigurableParamTest, GetDefaultOptionsTest) {
TestConfigureOptions(config_options_);
}
TEST_P(ConfigurableParamTest, ConfigureFromPropsTest) {
std::string opt_str, mismatch;
std::unordered_set<std::string> names;
std::unique_ptr<Configurable> copy(factory_());
ASSERT_OK(object_->ConfigureFromString(config_options_, configuration_));
config_options_.delimiter = "\n";
ASSERT_OK(object_->GetOptionString(config_options_, &opt_str));
std::istringstream iss(opt_str);
std::unordered_map<std::string, std::string> copy_map;
std::string line;
for (int line_num = 0; std::getline(iss, line); line_num++) {
std::string name;
std::string value;
ASSERT_OK(
RocksDBOptionsParser::ParseStatement(&name, &value, line, line_num));
copy_map[name] = value;
}
ASSERT_OK(copy->ConfigureFromMap(config_options_, copy_map));
ASSERT_TRUE(object_->AreEquivalent(config_options_, copy.get(), &mismatch));
}
static Configurable* SimpleFactory() {
return SimpleConfigurable::Create("simple");
}
static Configurable* UniqueFactory() {
return SimpleConfigurable::Create(
"simple", TestConfigMode::kSimpleMode | TestConfigMode::kUniqueMode);
}
static Configurable* SharedFactory() {
return SimpleConfigurable::Create(
"simple", TestConfigMode::kSimpleMode | TestConfigMode::kSharedMode);
}
static Configurable* NestedFactory() {
return SimpleConfigurable::Create(
"simple", TestConfigMode::kSimpleMode | TestConfigMode::kNestedMode);
}
static Configurable* MutableFactory() {
return SimpleConfigurable::Create("simple", TestConfigMode::kMutableMode |
TestConfigMode::kSimpleMode |
TestConfigMode::kNestedMode);
}
static Configurable* ThreeWrappedFactory() {
std::shared_ptr<Configurable> child;
child.reset(
SimpleConfigurable::Create("child", TestConfigMode::kDefaultMode));
std::shared_ptr<Configurable> parent;
parent.reset(
new WrappedConfigurable("parent", TestConfigMode::kDefaultMode, child));
return new WrappedConfigurable("master", TestConfigMode::kDefaultMode,
parent);
}
static Configurable* ThreeDeepFactory() {
Configurable* simple = SimpleConfigurable::Create(
"Simple", TestConfigMode::kUniqueMode | TestConfigMode::kDefaultMode);
auto* unique =
simple->GetOptions<std::unique_ptr<Configurable>>("SimpleUnique");
unique->reset(SimpleConfigurable::Create(
"Child", TestConfigMode::kUniqueMode | TestConfigMode::kDefaultMode));
unique =
unique->get()->GetOptions<std::unique_ptr<Configurable>>("ChildUnique");
unique->reset(
SimpleConfigurable::Create("Child", TestConfigMode::kDefaultMode));
return simple;
}
static Configurable* DBOptionsFactory() {
auto config = DBOptionsAsConfigurable(DBOptions());
return config.release();
}
static Configurable* CFOptionsFactory() {
auto config = CFOptionsAsConfigurable(ColumnFamilyOptions());
return config.release();
}
static Configurable* BlockBasedFactory() { return NewBlockBasedTableFactory(); }
INSTANTIATE_TEST_CASE_P(
ParamTest, ConfigurableParamTest,
testing::Values(
std::pair<std::string, ConfigTestFactoryFunc>(
"int=42;bool=true;string=s", SimpleFactory),
std::pair<std::string, ConfigTestFactoryFunc>(
"int=42;unique={int=33;string=unique}", MutableFactory),
std::pair<std::string, ConfigTestFactoryFunc>(
"struct={int=33;bool=true;string=s;}", SimpleStructFactory),
std::pair<std::string, ConfigTestFactoryFunc>(
"int=33;bool=true;string=outer;"
"shared={int=42;string=shared}",
SharedFactory),
std::pair<std::string, ConfigTestFactoryFunc>(
"int=33;bool=true;string=outer;"
"unique={int=42;string=unique}",
UniqueFactory),
std::pair<std::string, ConfigTestFactoryFunc>(
"int=11;bool=true;string=outer;"
"pointer={int=22;string=pointer};"
"unique={int=33;string=unique};"
"shared={int=44;string=shared}",
NestedFactory),
std::pair<std::string, ConfigTestFactoryFunc>(
"int=11;bool=true;string=outer;"
"inner={int=22;string=parent;"
"inner={int=33;string=child}};",
ThreeWrappedFactory),
std::pair<std::string, ConfigTestFactoryFunc>(
"int=11;bool=true;string=outer;"
"unique={int=22;string=inner;"
"unique={int=33;string=unique}};",
ThreeDeepFactory),
std::pair<std::string, ConfigTestFactoryFunc>("max_background_jobs=100;"
"max_open_files=200;",
DBOptionsFactory),
std::pair<std::string, ConfigTestFactoryFunc>(
"table_factory=BlockBasedTable;"
"disable_auto_compactions=true;",
CFOptionsFactory),
std::pair<std::string, ConfigTestFactoryFunc>("block_size=1024;"
"no_block_cache=true;",
BlockBasedFactory)));
#endif // ROCKSDB_LITE
} // namespace test
} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
#ifdef GFLAGS
ParseCommandLineFlags(&argc, &argv, true);
#endif // GFLAGS
return RUN_ALL_TESTS();
}

@ -0,0 +1,127 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <algorithm>
#include <memory>
#include <unordered_map>
#include "options/configurable_helper.h"
#include "rocksdb/configurable.h"
#include "rocksdb/utilities/options_type.h"
namespace ROCKSDB_NAMESPACE {
struct ColumnFamilyOptions;
struct DBOptions;
namespace test {
enum TestEnum { kTestA, kTestB };
static const std::unordered_map<std::string, int> test_enum_map = {
{"A", TestEnum::kTestA},
{"B", TestEnum::kTestB},
};
struct TestOptions {
int i = 0;
bool b = false;
bool d = true;
TestEnum e = TestEnum::kTestA;
std::string s = "";
std::string u = "";
};
static std::unordered_map<std::string, OptionTypeInfo> simple_option_info = {
#ifndef ROCKSDB_LITE
{"int",
{offsetof(struct TestOptions, i), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
{"bool",
{offsetof(struct TestOptions, b), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"string",
{offsetof(struct TestOptions, s), OptionType::kString,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
#endif // ROCKSDB_LITE
};
static std::unordered_map<std::string, OptionTypeInfo> enum_option_info = {
#ifndef ROCKSDB_LITE
{"enum",
OptionTypeInfo::Enum(offsetof(struct TestOptions, e), &test_enum_map)}
#endif
};
static std::unordered_map<std::string, OptionTypeInfo> unique_option_info = {
#ifndef ROCKSDB_LITE
{"unique",
{0, OptionType::kConfigurable, OptionVerificationType::kNormal,
(OptionTypeFlags::kUnique | OptionTypeFlags::kMutable)}},
#endif // ROCKSDB_LITE
};
static std::unordered_map<std::string, OptionTypeInfo> shared_option_info = {
#ifndef ROCKSDB_LITE
{"shared",
{0, OptionType::kConfigurable, OptionVerificationType::kNormal,
(OptionTypeFlags::kShared)}},
#endif // ROCKSDB_LITE
};
static std::unordered_map<std::string, OptionTypeInfo> pointer_option_info = {
#ifndef ROCKSDB_LITE
{"pointer",
{0, OptionType::kConfigurable, OptionVerificationType::kNormal,
OptionTypeFlags::kRawPointer}},
#endif // ROCKSDB_LITE
};
enum TestConfigMode {
kEmptyMode = 0x0, // Don't register anything
kMutableMode = 0x01, // Configuration is mutable
kSimpleMode = 0x02, // Use the simple options
kEnumMode = 0x04, // Use the enum options
kDefaultMode = kSimpleMode, // Use no inner nested configurations
kSharedMode = 0x10, // Use shared configuration
kUniqueMode = 0x20, // Use unique configuration
kRawPtrMode = 0x40, // Use pointer configuration
kNestedMode = (kSharedMode | kUniqueMode | kRawPtrMode),
kAllOptMode = (kNestedMode | kEnumMode | kSimpleMode),
};
template <typename T>
class TestConfigurable : public Configurable {
protected:
std::string name_;
std::string prefix_;
TestOptions options_;
public:
std::unique_ptr<T> unique_;
std::shared_ptr<T> shared_;
T* pointer_;
TestConfigurable(const std::string& name, int mode,
const std::unordered_map<std::string, OptionTypeInfo>* map =
&simple_option_info)
: name_(name), pointer_(nullptr) {
prefix_ = "test." + name + ".";
if ((mode & TestConfigMode::kSimpleMode) != 0) {
ConfigurableHelper::RegisterOptions(*this, name_, &options_, map);
}
if ((mode & TestConfigMode::kEnumMode) != 0) {
ConfigurableHelper::RegisterOptions(*this, name_ + "Enum", &options_,
&enum_option_info);
}
}
~TestConfigurable() override { delete pointer_; }
};
} // namespace test
} // namespace ROCKSDB_NAMESPACE

@ -8,14 +8,18 @@
#include <cinttypes> #include <cinttypes>
#include "logging/logging.h" #include "logging/logging.h"
#include "options/configurable_helper.h"
#include "options/options_helper.h" #include "options/options_helper.h"
#include "options/options_parser.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/cache.h" #include "rocksdb/configurable.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/file_system.h" #include "rocksdb/file_system.h"
#include "rocksdb/rate_limiter.h" #include "rocksdb/rate_limiter.h"
#include "rocksdb/sst_file_manager.h" #include "rocksdb/sst_file_manager.h"
#include "rocksdb/utilities/options_type.h"
#include "rocksdb/wal_filter.h" #include "rocksdb/wal_filter.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
@ -42,8 +46,87 @@ static std::unordered_map<std::string, InfoLogLevel> info_log_level_string_map =
{"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL}, {"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL},
{"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}}; {"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}};
std::unordered_map<std::string, OptionTypeInfo> static std::unordered_map<std::string, OptionTypeInfo>
OptionsHelper::db_options_type_info = { db_mutable_options_type_info = {
{"allow_os_buffer",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable}},
{"max_background_jobs",
{offsetof(struct MutableDBOptions, max_background_jobs),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_background_compactions",
{offsetof(struct MutableDBOptions, max_background_compactions),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"base_background_compactions",
{offsetof(struct MutableDBOptions, base_background_compactions),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_subcompactions",
{offsetof(struct MutableDBOptions, max_subcompactions),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"avoid_flush_during_shutdown",
{offsetof(struct MutableDBOptions, avoid_flush_during_shutdown),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"writable_file_max_buffer_size",
{offsetof(struct MutableDBOptions, writable_file_max_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"delayed_write_rate",
{offsetof(struct MutableDBOptions, delayed_write_rate),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_total_wal_size",
{offsetof(struct MutableDBOptions, max_total_wal_size),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"delete_obsolete_files_period_micros",
{offsetof(struct MutableDBOptions,
delete_obsolete_files_period_micros),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"stats_dump_period_sec",
{offsetof(struct MutableDBOptions, stats_dump_period_sec),
OptionType::kUInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"stats_persist_period_sec",
{offsetof(struct MutableDBOptions, stats_persist_period_sec),
OptionType::kUInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"stats_history_buffer_size",
{offsetof(struct MutableDBOptions, stats_history_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_open_files",
{offsetof(struct MutableDBOptions, max_open_files), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
{"bytes_per_sync",
{offsetof(struct MutableDBOptions, bytes_per_sync),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"wal_bytes_per_sync",
{offsetof(struct MutableDBOptions, wal_bytes_per_sync),
OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"strict_bytes_per_sync",
{offsetof(struct MutableDBOptions, strict_bytes_per_sync),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"compaction_readahead_size",
{offsetof(struct MutableDBOptions, compaction_readahead_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
{"max_background_flushes",
{offsetof(struct MutableDBOptions, max_background_flushes),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable}},
};
static std::unordered_map<std::string, OptionTypeInfo>
db_immutable_options_type_info = {
/* /*
// not yet supported // not yet supported
std::shared_ptr<Cache> row_cache; std::shared_ptr<Cache> row_cache;
@ -55,321 +138,259 @@ std::unordered_map<std::string, OptionTypeInfo>
std::vector<std::shared_ptr<EventListener>> listeners; std::vector<std::shared_ptr<EventListener>> listeners;
*/ */
{"advise_random_on_open", {"advise_random_on_open",
{offsetof(struct DBOptions, advise_random_on_open), {offsetof(struct ImmutableDBOptions, advise_random_on_open),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"allow_mmap_reads", {"allow_mmap_reads",
{offsetof(struct DBOptions, allow_mmap_reads), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, allow_mmap_reads),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"allow_fallocate", {"allow_fallocate",
{offsetof(struct DBOptions, allow_fallocate), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, allow_fallocate),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"allow_mmap_writes", {"allow_mmap_writes",
{offsetof(struct DBOptions, allow_mmap_writes), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, allow_mmap_writes),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"use_direct_reads", {"use_direct_reads",
{offsetof(struct DBOptions, use_direct_reads), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, use_direct_reads),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"use_direct_writes", {"use_direct_writes",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"use_direct_io_for_flush_and_compaction", {"use_direct_io_for_flush_and_compaction",
{offsetof(struct DBOptions, use_direct_io_for_flush_and_compaction), {offsetof(struct ImmutableDBOptions,
use_direct_io_for_flush_and_compaction),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"allow_2pc", {"allow_2pc",
{offsetof(struct DBOptions, allow_2pc), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, allow_2pc), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"allow_os_buffer",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kMutable, 0}},
{"create_if_missing", {"create_if_missing",
{offsetof(struct DBOptions, create_if_missing), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, create_if_missing),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"create_missing_column_families", {"create_missing_column_families",
{offsetof(struct DBOptions, create_missing_column_families), {offsetof(struct ImmutableDBOptions, create_missing_column_families),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"disableDataSync", {"disableDataSync",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"disable_data_sync", // for compatibility {"disable_data_sync", // for compatibility
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"enable_thread_tracking", {"enable_thread_tracking",
{offsetof(struct DBOptions, enable_thread_tracking), {offsetof(struct ImmutableDBOptions, enable_thread_tracking),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"error_if_exists", {"error_if_exists",
{offsetof(struct DBOptions, error_if_exists), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, error_if_exists),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"is_fd_close_on_exec", {"is_fd_close_on_exec",
{offsetof(struct DBOptions, is_fd_close_on_exec), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, is_fd_close_on_exec),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"paranoid_checks", {"paranoid_checks",
{offsetof(struct DBOptions, paranoid_checks), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, paranoid_checks),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"skip_log_error_on_recovery",
{offsetof(struct DBOptions, skip_log_error_on_recovery),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"skip_log_error_on_recovery",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
{"skip_stats_update_on_db_open", {"skip_stats_update_on_db_open",
{offsetof(struct DBOptions, skip_stats_update_on_db_open), {offsetof(struct ImmutableDBOptions, skip_stats_update_on_db_open),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"skip_checking_sst_file_sizes_on_db_open", {"skip_checking_sst_file_sizes_on_db_open",
{offsetof(struct DBOptions, skip_checking_sst_file_sizes_on_db_open), {offsetof(struct ImmutableDBOptions,
skip_checking_sst_file_sizes_on_db_open),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"new_table_reader_for_compaction_inputs", {"new_table_reader_for_compaction_inputs",
{offsetof(struct DBOptions, new_table_reader_for_compaction_inputs), {offsetof(struct ImmutableDBOptions,
new_table_reader_for_compaction_inputs),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"compaction_readahead_size",
{offsetof(struct DBOptions, compaction_readahead_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, compaction_readahead_size)}},
{"random_access_max_buffer_size", {"random_access_max_buffer_size",
{offsetof(struct DBOptions, random_access_max_buffer_size), {offsetof(struct ImmutableDBOptions, random_access_max_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"use_adaptive_mutex", {"use_adaptive_mutex",
{offsetof(struct DBOptions, use_adaptive_mutex), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, use_adaptive_mutex),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"use_fsync", {"use_fsync",
{offsetof(struct DBOptions, use_fsync), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, use_fsync), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"max_background_jobs",
{offsetof(struct DBOptions, max_background_jobs), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_background_jobs)}},
{"max_background_compactions",
{offsetof(struct DBOptions, max_background_compactions),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_background_compactions)}},
{"max_subcompactions",
{offsetof(struct DBOptions, max_subcompactions), OptionType::kUInt32T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_subcompactions)}},
{"base_background_compactions",
{offsetof(struct DBOptions, base_background_compactions),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, base_background_compactions)}},
{"max_background_flushes",
{offsetof(struct DBOptions, max_background_flushes), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_background_flushes)}},
{"max_file_opening_threads", {"max_file_opening_threads",
{offsetof(struct DBOptions, max_file_opening_threads), {offsetof(struct ImmutableDBOptions, max_file_opening_threads),
OptionType::kInt, OptionVerificationType::kNormal, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"max_open_files",
{offsetof(struct DBOptions, max_open_files), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_open_files)}},
{"table_cache_numshardbits", {"table_cache_numshardbits",
{offsetof(struct DBOptions, table_cache_numshardbits), {offsetof(struct ImmutableDBOptions, table_cache_numshardbits),
OptionType::kInt, OptionVerificationType::kNormal, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"db_write_buffer_size", {"db_write_buffer_size",
{offsetof(struct DBOptions, db_write_buffer_size), OptionType::kSizeT, {offsetof(struct ImmutableDBOptions, db_write_buffer_size),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"keep_log_file_num", {"keep_log_file_num",
{offsetof(struct DBOptions, keep_log_file_num), OptionType::kSizeT, {offsetof(struct ImmutableDBOptions, keep_log_file_num),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"recycle_log_file_num", {"recycle_log_file_num",
{offsetof(struct DBOptions, recycle_log_file_num), OptionType::kSizeT, {offsetof(struct ImmutableDBOptions, recycle_log_file_num),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"log_file_time_to_roll", {"log_file_time_to_roll",
{offsetof(struct DBOptions, log_file_time_to_roll), OptionType::kSizeT, {offsetof(struct ImmutableDBOptions, log_file_time_to_roll),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"manifest_preallocation_size", {"manifest_preallocation_size",
{offsetof(struct DBOptions, manifest_preallocation_size), {offsetof(struct ImmutableDBOptions, manifest_preallocation_size),
OptionType::kSizeT, OptionVerificationType::kNormal, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"max_log_file_size", {"max_log_file_size",
{offsetof(struct DBOptions, max_log_file_size), OptionType::kSizeT, {offsetof(struct ImmutableDBOptions, max_log_file_size),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"db_log_dir", {"db_log_dir",
{offsetof(struct DBOptions, db_log_dir), OptionType::kString, {offsetof(struct ImmutableDBOptions, db_log_dir), OptionType::kString,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"wal_dir", {"wal_dir",
{offsetof(struct DBOptions, wal_dir), OptionType::kString, {offsetof(struct ImmutableDBOptions, wal_dir), OptionType::kString,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"WAL_size_limit_MB", {"WAL_size_limit_MB",
{offsetof(struct DBOptions, WAL_size_limit_MB), OptionType::kUInt64T, {offsetof(struct ImmutableDBOptions, wal_size_limit_mb),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"WAL_ttl_seconds", {"WAL_ttl_seconds",
{offsetof(struct DBOptions, WAL_ttl_seconds), OptionType::kUInt64T, {offsetof(struct ImmutableDBOptions, wal_ttl_seconds),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"bytes_per_sync",
{offsetof(struct DBOptions, bytes_per_sync), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, bytes_per_sync)}},
{"delayed_write_rate",
{offsetof(struct DBOptions, delayed_write_rate), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, delayed_write_rate)}},
{"delete_obsolete_files_period_micros",
{offsetof(struct DBOptions, delete_obsolete_files_period_micros),
OptionType::kUInt64T, OptionVerificationType::kNormal, OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable, OptionTypeFlags::kNone}},
offsetof(struct MutableDBOptions,
delete_obsolete_files_period_micros)}},
{"max_manifest_file_size", {"max_manifest_file_size",
{offsetof(struct DBOptions, max_manifest_file_size), {offsetof(struct ImmutableDBOptions, max_manifest_file_size),
OptionType::kUInt64T, OptionVerificationType::kNormal, OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"max_total_wal_size",
{offsetof(struct DBOptions, max_total_wal_size), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, max_total_wal_size)}},
{"wal_bytes_per_sync",
{offsetof(struct DBOptions, wal_bytes_per_sync), OptionType::kUInt64T,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, wal_bytes_per_sync)}},
{"strict_bytes_per_sync",
{offsetof(struct DBOptions, strict_bytes_per_sync),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, strict_bytes_per_sync)}},
{"stats_dump_period_sec",
{offsetof(struct DBOptions, stats_dump_period_sec), OptionType::kUInt,
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, stats_dump_period_sec)}},
{"stats_persist_period_sec",
{offsetof(struct DBOptions, stats_persist_period_sec),
OptionType::kUInt, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, stats_persist_period_sec)}},
{"persist_stats_to_disk", {"persist_stats_to_disk",
{offsetof(struct DBOptions, persist_stats_to_disk), {offsetof(struct ImmutableDBOptions, persist_stats_to_disk),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, OptionTypeFlags::kNone}},
offsetof(struct ImmutableDBOptions, persist_stats_to_disk)}},
{"stats_history_buffer_size",
{offsetof(struct DBOptions, stats_history_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, stats_history_buffer_size)}},
{"fail_if_options_file_error", {"fail_if_options_file_error",
{offsetof(struct DBOptions, fail_if_options_file_error), {offsetof(struct ImmutableDBOptions, fail_if_options_file_error),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"enable_pipelined_write", {"enable_pipelined_write",
{offsetof(struct DBOptions, enable_pipelined_write), {offsetof(struct ImmutableDBOptions, enable_pipelined_write),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"unordered_write", {"unordered_write",
{offsetof(struct DBOptions, unordered_write), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, unordered_write),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"allow_concurrent_memtable_write", {"allow_concurrent_memtable_write",
{offsetof(struct DBOptions, allow_concurrent_memtable_write), {offsetof(struct ImmutableDBOptions, allow_concurrent_memtable_write),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"wal_recovery_mode", OptionTypeInfo::Enum<WALRecoveryMode>( {"wal_recovery_mode",
offsetof(struct DBOptions, wal_recovery_mode), OptionTypeInfo::Enum<WALRecoveryMode>(
offsetof(struct ImmutableDBOptions, wal_recovery_mode),
&wal_recovery_mode_string_map)}, &wal_recovery_mode_string_map)},
{"enable_write_thread_adaptive_yield", {"enable_write_thread_adaptive_yield",
{offsetof(struct DBOptions, enable_write_thread_adaptive_yield), {offsetof(struct ImmutableDBOptions,
enable_write_thread_adaptive_yield),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"write_thread_slow_yield_usec", {"write_thread_slow_yield_usec",
{offsetof(struct DBOptions, write_thread_slow_yield_usec), {offsetof(struct ImmutableDBOptions, write_thread_slow_yield_usec),
OptionType::kUInt64T, OptionVerificationType::kNormal, OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"max_write_batch_group_size_bytes", {"max_write_batch_group_size_bytes",
{offsetof(struct DBOptions, max_write_batch_group_size_bytes), {offsetof(struct ImmutableDBOptions, max_write_batch_group_size_bytes),
OptionType::kUInt64T, OptionVerificationType::kNormal, OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"write_thread_max_yield_usec", {"write_thread_max_yield_usec",
{offsetof(struct DBOptions, write_thread_max_yield_usec), {offsetof(struct ImmutableDBOptions, write_thread_max_yield_usec),
OptionType::kUInt64T, OptionVerificationType::kNormal, OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"access_hint_on_compaction_start", {"access_hint_on_compaction_start",
OptionTypeInfo::Enum<DBOptions::AccessHint>( OptionTypeInfo::Enum<DBOptions::AccessHint>(
offsetof(struct DBOptions, access_hint_on_compaction_start), offsetof(struct ImmutableDBOptions,
access_hint_on_compaction_start),
&access_hint_string_map)}, &access_hint_string_map)},
{"info_log_level", OptionTypeInfo::Enum<InfoLogLevel>( {"info_log_level",
offsetof(struct DBOptions, info_log_level), OptionTypeInfo::Enum<InfoLogLevel>(
offsetof(struct ImmutableDBOptions, info_log_level),
&info_log_level_string_map)}, &info_log_level_string_map)},
{"dump_malloc_stats", {"dump_malloc_stats",
{offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, dump_malloc_stats),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}},
{"avoid_flush_during_recovery",
{offsetof(struct DBOptions, avoid_flush_during_recovery),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"avoid_flush_during_shutdown", {"avoid_flush_during_recovery",
{offsetof(struct DBOptions, avoid_flush_during_shutdown), {offsetof(struct ImmutableDBOptions, avoid_flush_during_recovery),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable, OptionTypeFlags::kNone}},
offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}},
{"writable_file_max_buffer_size",
{offsetof(struct DBOptions, writable_file_max_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable,
offsetof(struct MutableDBOptions, writable_file_max_buffer_size)}},
{"allow_ingest_behind", {"allow_ingest_behind",
{offsetof(struct DBOptions, allow_ingest_behind), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, allow_ingest_behind),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, OptionType::kBoolean, OptionVerificationType::kNormal,
offsetof(struct ImmutableDBOptions, allow_ingest_behind)}}, OptionTypeFlags::kNone}},
{"preserve_deletes", {"preserve_deletes",
{offsetof(struct DBOptions, preserve_deletes), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, preserve_deletes),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, OptionType::kBoolean, OptionVerificationType::kNormal,
offsetof(struct ImmutableDBOptions, preserve_deletes)}}, OptionTypeFlags::kNone}},
{"concurrent_prepare", // Deprecated by two_write_queues {"concurrent_prepare", // Deprecated by two_write_queues
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"two_write_queues", {"two_write_queues",
{offsetof(struct DBOptions, two_write_queues), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, two_write_queues),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, OptionType::kBoolean, OptionVerificationType::kNormal,
offsetof(struct ImmutableDBOptions, two_write_queues)}}, OptionTypeFlags::kNone}},
{"manual_wal_flush", {"manual_wal_flush",
{offsetof(struct DBOptions, manual_wal_flush), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, manual_wal_flush),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, OptionType::kBoolean, OptionVerificationType::kNormal,
offsetof(struct ImmutableDBOptions, manual_wal_flush)}}, OptionTypeFlags::kNone}},
{"seq_per_batch", {"seq_per_batch",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"atomic_flush", {"atomic_flush",
{offsetof(struct DBOptions, atomic_flush), OptionType::kBoolean, {offsetof(struct ImmutableDBOptions, atomic_flush),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, OptionType::kBoolean, OptionVerificationType::kNormal,
offsetof(struct ImmutableDBOptions, atomic_flush)}}, OptionTypeFlags::kNone}},
{"avoid_unnecessary_blocking_io", {"avoid_unnecessary_blocking_io",
{offsetof(struct DBOptions, avoid_unnecessary_blocking_io), {offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, OptionTypeFlags::kNone}},
offsetof(struct ImmutableDBOptions, avoid_unnecessary_blocking_io)}},
{"write_dbid_to_manifest", {"write_dbid_to_manifest",
{offsetof(struct DBOptions, write_dbid_to_manifest), {offsetof(struct ImmutableDBOptions, write_dbid_to_manifest),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"log_readahead_size", {"log_readahead_size",
{offsetof(struct DBOptions, log_readahead_size), OptionType::kSizeT, {offsetof(struct ImmutableDBOptions, log_readahead_size),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"best_efforts_recovery", {"best_efforts_recovery",
{offsetof(struct DBOptions, best_efforts_recovery), {offsetof(struct ImmutableDBOptions, best_efforts_recovery),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"max_bgerror_resume_count", {"max_bgerror_resume_count",
{offsetof(struct DBOptions, max_bgerror_resume_count), {offsetof(struct ImmutableDBOptions, max_bgerror_resume_count),
OptionType::kInt, OptionVerificationType::kNormal, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"bgerror_resume_retry_interval", {"bgerror_resume_retry_interval",
{offsetof(struct DBOptions, bgerror_resume_retry_interval), {offsetof(struct ImmutableDBOptions, bgerror_resume_retry_interval),
OptionType::kUInt64T, OptionVerificationType::kNormal, OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
// The following properties were handled as special cases in ParseOption // The following properties were handled as special cases in ParseOption
// This means that the properties could be read from the options file // This means that the properties could be read from the options file
// but never written to the file or compared to each other. // but never written to the file or compared to each other.
{"rate_limiter_bytes_per_sec", {"rate_limiter_bytes_per_sec",
{offsetof(struct DBOptions, rate_limiter), OptionType::kUnknown, {offsetof(struct ImmutableDBOptions, rate_limiter),
OptionVerificationType::kNormal, OptionType::kUnknown, OptionVerificationType::kNormal,
(OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), 0, (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever),
// Parse the input value as a RateLimiter // Parse the input value as a RateLimiter
[](const ConfigOptions& /*opts*/, const std::string& /*name*/, [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const std::string& value, char* addr) { const std::string& value, char* addr) {
@ -380,9 +401,9 @@ std::unordered_map<std::string, OptionTypeInfo>
return Status::OK(); return Status::OK();
}}}, }}},
{"env", {"env",
{offsetof(struct DBOptions, env), OptionType::kUnknown, {offsetof(struct ImmutableDBOptions, env), OptionType::kUnknown,
OptionVerificationType::kNormal, OptionVerificationType::kNormal,
(OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever), 0, (OptionTypeFlags::kDontSerialize | OptionTypeFlags::kCompareNever),
// Parse the input value as an Env // Parse the input value as an Env
[](const ConfigOptions& /*opts*/, const std::string& /*name*/, [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
const std::string& value, char* addr) { const std::string& value, char* addr) {
@ -395,6 +416,74 @@ std::unordered_map<std::string, OptionTypeInfo>
return s; return s;
}}}, }}},
}; };
const std::string OptionsHelper::kDBOptionsName = "DBOptions";
class MutableDBConfigurable : public Configurable {
public:
MutableDBConfigurable(const MutableDBOptions& mdb) {
mutable_ = mdb;
ConfigurableHelper::RegisterOptions(*this, &mutable_,
&db_mutable_options_type_info);
}
protected:
MutableDBOptions mutable_;
};
class DBOptionsConfigurable : public MutableDBConfigurable {
public:
DBOptionsConfigurable(const DBOptions& opts)
: MutableDBConfigurable(MutableDBOptions(opts)), db_options_(opts) {
// The ImmutableDBOptions currently requires the env to be non-null. Make
// sure it is
if (opts.env != nullptr) {
immutable_ = ImmutableDBOptions(opts);
} else {
DBOptions copy = opts;
copy.env = Env::Default();
immutable_ = ImmutableDBOptions(copy);
}
ConfigurableHelper::RegisterOptions(*this, &immutable_,
&db_immutable_options_type_info);
}
protected:
Status ConfigureOptions(
const ConfigOptions& config_options,
const std::unordered_map<std::string, std::string>& opts_map,
std::unordered_map<std::string, std::string>* unused) override {
Status s = ConfigurableHelper::ConfigureOptions(config_options, *this,
opts_map, unused);
if (s.ok()) {
db_options_ = BuildDBOptions(immutable_, mutable_);
s = PrepareOptions(config_options);
}
return s;
}
const void* GetOptionsPtr(const std::string& name) const override {
if (name == OptionsHelper::kDBOptionsName) {
return &db_options_;
} else {
return MutableDBConfigurable::GetOptionsPtr(name);
}
}
private:
ImmutableDBOptions immutable_;
DBOptions db_options_;
};
std::unique_ptr<Configurable> DBOptionsAsConfigurable(
const MutableDBOptions& opts) {
std::unique_ptr<Configurable> ptr(new MutableDBConfigurable(opts));
return ptr;
}
std::unique_ptr<Configurable> DBOptionsAsConfigurable(const DBOptions& opts) {
std::unique_ptr<Configurable> ptr(new DBOptionsConfigurable(opts));
return ptr;
}
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {} ImmutableDBOptions::ImmutableDBOptions() : ImmutableDBOptions(Options()) {}

@ -13,6 +13,7 @@
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
struct ImmutableDBOptions { struct ImmutableDBOptions {
static const char* kName() { return "ImmutableDBOptions"; }
ImmutableDBOptions(); ImmutableDBOptions();
explicit ImmutableDBOptions(const DBOptions& options); explicit ImmutableDBOptions(const DBOptions& options);
@ -92,6 +93,7 @@ struct ImmutableDBOptions {
}; };
struct MutableDBOptions { struct MutableDBOptions {
static const char* kName() { return "MutableDBOptions"; }
MutableDBOptions(); MutableDBOptions();
explicit MutableDBOptions(const MutableDBOptions& options) = default; explicit MutableDBOptions(const MutableDBOptions& options) = default;
explicit MutableDBOptions(const DBOptions& options); explicit MutableDBOptions(const DBOptions& options);

@ -135,7 +135,7 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
ROCKS_LOG_HEADER(log, " Options.table_factory: %s", ROCKS_LOG_HEADER(log, " Options.table_factory: %s",
table_factory->Name()); table_factory->Name());
ROCKS_LOG_HEADER(log, " table_factory options: %s", ROCKS_LOG_HEADER(log, " table_factory options: %s",
table_factory->GetPrintableTableOptions().c_str()); table_factory->GetPrintableOptions().c_str());
ROCKS_LOG_HEADER(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt, ROCKS_LOG_HEADER(log, " Options.write_buffer_size: %" ROCKSDB_PRIszt,
write_buffer_size); write_buffer_size);
ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number: %d", ROCKS_LOG_HEADER(log, " Options.max_write_buffer_number: %d",

@ -10,11 +10,13 @@
#include <unordered_set> #include <unordered_set>
#include <vector> #include <vector>
#include "options/options_type.h" #include "options/cf_options.h"
#include "options/db_options.h"
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h" #include "rocksdb/compaction_filter.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/filter_policy.h" #include "rocksdb/filter_policy.h"
#include "rocksdb/flush_block_policy.h"
#include "rocksdb/memtablerep.h" #include "rocksdb/memtablerep.h"
#include "rocksdb/merge_operator.h" #include "rocksdb/merge_operator.h"
#include "rocksdb/options.h" #include "rocksdb/options.h"
@ -22,11 +24,23 @@
#include "rocksdb/slice_transform.h" #include "rocksdb/slice_transform.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/object_registry.h"
#include "table/block_based/block_based_table_factory.h" #include "rocksdb/utilities/options_type.h"
#include "table/plain/plain_table_factory.h"
#include "util/string_util.h" #include "util/string_util.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
Status ValidateOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) {
Status s;
#ifndef ROCKSDB_LITE
auto db_cfg = DBOptionsAsConfigurable(db_opts);
auto cf_cfg = CFOptionsAsConfigurable(cf_opts);
s = db_cfg->ValidateOptions(db_opts, cf_opts);
if (s.ok()) s = cf_cfg->ValidateOptions(db_opts, cf_opts);
#else
s = cf_opts.table_factory->ValidateOptions(db_opts, cf_opts);
#endif
return s;
}
DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
const MutableDBOptions& mutable_db_options) { const MutableDBOptions& mutable_db_options) {
@ -336,7 +350,7 @@ bool ParseSliceTransform(
return false; return false;
} }
bool ParseOptionHelper(char* opt_address, const OptionType& opt_type, static bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
const std::string& value) { const std::string& value) {
switch (opt_type) { switch (opt_type) {
case OptionType::kBoolean: case OptionType::kBoolean:
@ -472,14 +486,6 @@ bool SerializeSingleOptionHelper(const char* opt_address,
: kNullptrString; : kNullptrString;
break; break;
} }
case OptionType::kTableFactory: {
const auto* table_factory_ptr =
reinterpret_cast<const std::shared_ptr<const TableFactory>*>(
opt_address);
*value = table_factory_ptr->get() ? table_factory_ptr->get()->Name()
: kNullptrString;
break;
}
case OptionType::kComparator: { case OptionType::kComparator: {
// it's a const pointer of const Comparator* // it's a const pointer of const Comparator*
const auto* ptr = reinterpret_cast<const Comparator* const*>(opt_address); const auto* ptr = reinterpret_cast<const Comparator* const*>(opt_address);
@ -555,36 +561,29 @@ bool SerializeSingleOptionHelper(const char* opt_address,
return true; return true;
} }
template <typename T>
Status ConfigureFromMap(
const ConfigOptions& config_options,
const std::unordered_map<std::string, std::string>& opt_map,
const std::string& option_name, Configurable* config, T* new_opts) {
Status s = config->ConfigureFromMap(config_options, opt_map);
if (s.ok()) {
*new_opts = *(config->GetOptions<T>(option_name));
}
return s;
}
Status GetMutableOptionsFromStrings( Status GetMutableOptionsFromStrings(
const MutableCFOptions& base_options, const MutableCFOptions& base_options,
const std::unordered_map<std::string, std::string>& options_map, const std::unordered_map<std::string, std::string>& options_map,
Logger* info_log, MutableCFOptions* new_options) { Logger* /*info_log*/, MutableCFOptions* new_options) {
assert(new_options); assert(new_options);
*new_options = base_options; *new_options = base_options;
ConfigOptions config_options; ConfigOptions config_options;
for (const auto& o : options_map) { const auto config = CFOptionsAsConfigurable(base_options);
std::string elem; return ConfigureFromMap<MutableCFOptions>(config_options, options_map,
const auto opt_info = MutableCFOptions::kName(),
OptionTypeInfo::Find(o.first, cf_options_type_info, &elem); config.get(), new_options);
if (opt_info == nullptr) {
return Status::InvalidArgument("Unrecognized option: " + o.first);
} else if (!opt_info->IsMutable()) {
return Status::InvalidArgument("Option not changeable: " + o.first);
} else if (opt_info->IsDeprecated()) {
// log warning when user tries to set a deprecated option but don't fail
// the call for compatibility.
ROCKS_LOG_WARN(info_log, "%s is a deprecated option and cannot be set",
o.first.c_str());
} else {
Status s = opt_info->Parse(
config_options, elem, o.second,
reinterpret_cast<char*>(new_options) + opt_info->mutable_offset_);
if (!s.ok()) {
return s;
}
}
}
return Status::OK();
} }
Status GetMutableDBOptionsFromStrings( Status GetMutableDBOptionsFromStrings(
@ -595,29 +594,10 @@ Status GetMutableDBOptionsFromStrings(
*new_options = base_options; *new_options = base_options;
ConfigOptions config_options; ConfigOptions config_options;
for (const auto& o : options_map) { auto config = DBOptionsAsConfigurable(base_options);
try { return ConfigureFromMap<MutableDBOptions>(config_options, options_map,
std::string elem; MutableDBOptions::kName(),
const auto opt_info = config.get(), new_options);
OptionTypeInfo::Find(o.first, db_options_type_info, &elem);
if (opt_info == nullptr) {
return Status::InvalidArgument("Unrecognized option: " + o.first);
} else if (!opt_info->IsMutable()) {
return Status::InvalidArgument("Option not changeable: " + o.first);
} else {
Status s = opt_info->Parse(
config_options, elem, o.second,
reinterpret_cast<char*>(new_options) + opt_info->mutable_offset_);
if (!s.ok()) {
return s;
}
}
} catch (std::exception& e) {
return Status::InvalidArgument("Error parsing " + o.first + ":" +
std::string(e.what()));
}
}
return Status::OK();
} }
Status StringToMap(const std::string& opts_str, Status StringToMap(const std::string& opts_str,
@ -660,30 +640,11 @@ Status StringToMap(const std::string& opts_str,
return Status::OK(); return Status::OK();
} }
Status GetStringFromStruct( Status GetStringFromMutableDBOptions(const ConfigOptions& config_options,
const ConfigOptions& config_options, const void* const opt_ptr, const MutableDBOptions& mutable_opts,
const std::unordered_map<std::string, OptionTypeInfo>& type_info,
std::string* opt_string) { std::string* opt_string) {
assert(opt_string); auto config = DBOptionsAsConfigurable(mutable_opts);
opt_string->clear(); return config->GetOptionString(config_options, opt_string);
for (const auto& iter : type_info) {
const auto& opt_info = iter.second;
// If the option is no longer used in rocksdb and marked as deprecated,
// we skip it in the serialization.
if (opt_info.ShouldSerialize()) {
const char* opt_addr =
reinterpret_cast<const char*>(opt_ptr) + opt_info.offset_;
std::string value;
Status s =
opt_info.Serialize(config_options, iter.first, opt_addr, &value);
if (s.ok()) {
opt_string->append(iter.first + "=" + value + config_options.delimiter);
} else {
return s;
}
}
}
return Status::OK();
} }
Status GetStringFromDBOptions(std::string* opt_string, Status GetStringFromDBOptions(std::string* opt_string,
@ -694,11 +655,22 @@ Status GetStringFromDBOptions(std::string* opt_string,
return GetStringFromDBOptions(config_options, db_options, opt_string); return GetStringFromDBOptions(config_options, db_options, opt_string);
} }
Status GetStringFromDBOptions(const ConfigOptions& cfg_options, Status GetStringFromDBOptions(const ConfigOptions& config_options,
const DBOptions& db_options, const DBOptions& db_options,
std::string* opt_string) { std::string* opt_string) {
return GetStringFromStruct(cfg_options, &db_options, db_options_type_info, assert(opt_string);
opt_string); opt_string->clear();
auto config = DBOptionsAsConfigurable(db_options);
return config->GetOptionString(config_options, opt_string);
}
Status GetStringFromMutableCFOptions(const ConfigOptions& config_options,
const MutableCFOptions& mutable_opts,
std::string* opt_string) {
assert(opt_string);
opt_string->clear();
const auto config = CFOptionsAsConfigurable(mutable_opts);
return config->GetOptionString(config_options, opt_string);
} }
Status GetStringFromColumnFamilyOptions(std::string* opt_string, Status GetStringFromColumnFamilyOptions(std::string* opt_string,
@ -713,8 +685,8 @@ Status GetStringFromColumnFamilyOptions(std::string* opt_string,
Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options, Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options,
const ColumnFamilyOptions& cf_options, const ColumnFamilyOptions& cf_options,
std::string* opt_string) { std::string* opt_string) {
return GetStringFromStruct(config_options, &cf_options, cf_options_type_info, const auto config = CFOptionsAsConfigurable(cf_options);
opt_string); return config->GetOptionString(config_options, opt_string);
} }
Status GetStringFromCompressionType(std::string* compression_str, Status GetStringFromCompressionType(std::string* compression_str,
@ -728,24 +700,6 @@ Status GetStringFromCompressionType(std::string* compression_str,
} }
} }
static Status ParseDBOption(const ConfigOptions& config_options,
const std::string& name,
const std::string& org_value,
DBOptions* new_options) {
const std::string& value = config_options.input_strings_escaped
? UnescapeOptionString(org_value)
: org_value;
std::string elem;
const auto opt_info = OptionTypeInfo::Find(name, db_options_type_info, &elem);
if (opt_info == nullptr) {
return Status::InvalidArgument("Unrecognized option DBOptions:", name);
} else {
return opt_info->Parse(
config_options, elem, value,
reinterpret_cast<char*>(new_options) + opt_info->offset_);
}
}
Status GetColumnFamilyOptionsFromMap( Status GetColumnFamilyOptionsFromMap(
const ColumnFamilyOptions& base_options, const ColumnFamilyOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
@ -764,24 +718,13 @@ Status GetColumnFamilyOptionsFromMap(
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
ColumnFamilyOptions* new_options) { ColumnFamilyOptions* new_options) {
assert(new_options); assert(new_options);
*new_options = base_options; *new_options = base_options;
for (const auto& o : opts_map) {
auto s = const auto config = CFOptionsAsConfigurable(base_options);
ParseColumnFamilyOption(config_options, o.first, o.second, new_options); return ConfigureFromMap<ColumnFamilyOptions>(config_options, opts_map,
if (!s.ok()) { OptionsHelper::kCFOptionsName,
if (s.IsNotSupported()) { config.get(), new_options);
continue;
} else if (s.IsInvalidArgument() &&
config_options.ignore_unknown_options) {
continue;
} else {
// Restore "new_options" to the default "base_options".
*new_options = base_options;
return s;
}
}
}
return Status::OK();
} }
Status GetColumnFamilyOptionsFromString( Status GetColumnFamilyOptionsFromString(
@ -825,44 +768,12 @@ Status GetDBOptionsFromMap(
const ConfigOptions& config_options, const DBOptions& base_options, const ConfigOptions& config_options, const DBOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
DBOptions* new_options) { DBOptions* new_options) {
return GetDBOptionsFromMapInternal(config_options, base_options, opts_map,
new_options, nullptr);
}
Status GetDBOptionsFromMapInternal(
const ConfigOptions& config_options, const DBOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map,
DBOptions* new_options,
std::vector<std::string>* unsupported_options_names) {
assert(new_options); assert(new_options);
*new_options = base_options; *new_options = base_options;
if (unsupported_options_names) { auto config = DBOptionsAsConfigurable(base_options);
unsupported_options_names->clear(); return ConfigureFromMap<DBOptions>(config_options, opts_map,
} OptionsHelper::kDBOptionsName,
for (const auto& o : opts_map) { config.get(), new_options);
auto s = ParseDBOption(config_options, o.first, o.second, new_options);
if (!s.ok()) {
if (s.IsNotSupported()) {
// If the deserialization of the specified option is not supported
// and an output vector of unsupported_options is provided, then
// we log the name of the unsupported option and proceed.
if (unsupported_options_names != nullptr) {
unsupported_options_names->push_back(o.first);
}
// Note that we still return Status::OK in such case to maintain
// the backward compatibility in the old public API defined in
// rocksdb/convenience.h
} else if (s.IsInvalidArgument() &&
config_options.ignore_unknown_options) {
continue;
} else {
// Restore "new_options" to the default "base_options".
*new_options = base_options;
return s;
}
}
}
return Status::OK();
} }
Status GetDBOptionsFromString(const DBOptions& base_options, Status GetDBOptionsFromString(const DBOptions& base_options,
@ -903,65 +814,31 @@ Status GetOptionsFromString(const Options& base_options,
Status GetOptionsFromString(const ConfigOptions& config_options, Status GetOptionsFromString(const ConfigOptions& config_options,
const Options& base_options, const Options& base_options,
const std::string& opts_str, Options* new_options) { const std::string& opts_str, Options* new_options) {
ColumnFamilyOptions new_cf_options;
std::unordered_map<std::string, std::string> unused_opts;
std::unordered_map<std::string, std::string> opts_map; std::unordered_map<std::string, std::string> opts_map;
*new_options = base_options;
Status s = StringToMap(opts_str, &opts_map); Status s = StringToMap(opts_str, &opts_map);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
DBOptions new_db_options(base_options); auto config = DBOptionsAsConfigurable(base_options);
ColumnFamilyOptions new_cf_options(base_options); s = config->ConfigureFromMap(config_options, opts_map, &unused_opts);
for (const auto& o : opts_map) {
if (ParseDBOption(config_options, o.first, o.second, &new_db_options)
.ok()) {
} else if (ParseColumnFamilyOption(config_options, o.first, o.second,
&new_cf_options)
.ok()) {
} else {
return Status::InvalidArgument("Can't parse option " + o.first);
}
}
*new_options = Options(new_db_options, new_cf_options);
return Status::OK();
}
Status GetTableFactoryFromMap( if (s.ok()) {
const std::string& factory_name, DBOptions* new_db_options =
const std::unordered_map<std::string, std::string>& opt_map, config->GetOptions<DBOptions>(OptionsHelper::kDBOptionsName);
std::shared_ptr<TableFactory>* table_factory, bool ignore_unknown_options) { if (!unused_opts.empty()) {
ConfigOptions s = GetColumnFamilyOptionsFromMap(config_options, base_options,
config_options; // Use default for escaped(true) and check (exact) unused_opts, &new_cf_options);
config_options.ignore_unknown_options = ignore_unknown_options; if (s.ok()) {
return GetTableFactoryFromMap(config_options, factory_name, opt_map, *new_options = Options(*new_db_options, new_cf_options);
table_factory);
}
Status GetTableFactoryFromMap(
const ConfigOptions& config_options, const std::string& factory_name,
const std::unordered_map<std::string, std::string>& opt_map,
std::shared_ptr<TableFactory>* table_factory) {
Status s;
if (factory_name == BlockBasedTableFactory::kName) {
BlockBasedTableOptions bbt_opt;
s = GetBlockBasedTableOptionsFromMap(
config_options, BlockBasedTableOptions(), opt_map, &bbt_opt);
if (!s.ok()) {
return s;
} }
table_factory->reset(new BlockBasedTableFactory(bbt_opt)); } else {
return s; *new_options = Options(*new_db_options, base_options);
} else if (factory_name == PlainTableFactory::kName) {
PlainTableOptions pt_opt;
s = GetPlainTableOptionsFromMap(config_options, PlainTableOptions(),
opt_map, &pt_opt);
if (!s.ok()) {
return s;
} }
table_factory->reset(new PlainTableFactory(pt_opt));
return s;
} }
// Return OK for not supported table factories as TableFactory
// Deserialization is optional.
table_factory->reset();
return s; return s;
} }
@ -1043,18 +920,41 @@ Status OptionTypeInfo::NextToken(const std::string& opts, char delimiter,
Status OptionTypeInfo::Parse(const ConfigOptions& config_options, Status OptionTypeInfo::Parse(const ConfigOptions& config_options,
const std::string& opt_name, const std::string& opt_name,
const std::string& opt_value, const std::string& value, void* opt_ptr) const {
char* opt_addr) const {
if (IsDeprecated()) { if (IsDeprecated()) {
return Status::OK(); return Status::OK();
} }
try { try {
char* opt_addr = reinterpret_cast<char*>(opt_ptr) + offset_;
const std::string& opt_value = config_options.input_strings_escaped
? UnescapeOptionString(value)
: value;
if (opt_addr == nullptr) { if (opt_addr == nullptr) {
return Status::NotFound("Could not find option", opt_name); return Status::NotFound("Could not find option", opt_name);
} else if (parse_func_ != nullptr) { } else if (parse_func_ != nullptr) {
return parse_func_(config_options, opt_name, opt_value, opt_addr); ConfigOptions copy = config_options;
copy.invoke_prepare_options = false;
return parse_func_(copy, opt_name, opt_value, opt_addr);
} else if (ParseOptionHelper(opt_addr, type_, opt_value)) { } else if (ParseOptionHelper(opt_addr, type_, opt_value)) {
return Status::OK(); return Status::OK();
} else if (IsConfigurable()) {
// The option is <config>.<name>
Configurable* config = AsRawPointer<Configurable>(opt_ptr);
if (opt_value.empty()) {
return Status::OK();
} else if (config == nullptr) {
return Status::NotFound("Could not find configurable: ", opt_name);
} else {
ConfigOptions copy = config_options;
copy.ignore_unknown_options = false;
copy.invoke_prepare_options = false;
if (opt_value.find("=") != std::string::npos) {
return config->ConfigureFromString(copy, opt_value);
} else {
return config->ConfigureOption(copy, opt_name, opt_value);
}
}
} else if (IsByName()) { } else if (IsByName()) {
return Status::NotSupported("Deserializing the option " + opt_name + return Status::NotSupported("Deserializing the option " + opt_name +
" is not supported"); " is not supported");
@ -1083,9 +983,8 @@ Status OptionTypeInfo::ParseStruct(
} }
const auto iter = struct_map->find(map_iter.first); const auto iter = struct_map->find(map_iter.first);
if (iter != struct_map->end()) { if (iter != struct_map->end()) {
status = status = iter->second.Parse(config_options, map_iter.first,
iter->second.Parse(config_options, map_iter.first, map_iter.second, map_iter.second, opt_addr);
opt_addr + iter->second.offset_);
} else { } else {
status = Status::InvalidArgument("Unrecognized option", status = Status::InvalidArgument("Unrecognized option",
struct_name + "." + map_iter.first); struct_name + "." + map_iter.first);
@ -1097,8 +996,7 @@ Status OptionTypeInfo::ParseStruct(
const auto opt_info = const auto opt_info =
Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
if (opt_info != nullptr) { if (opt_info != nullptr) {
status = opt_info->Parse(config_options, elem_name, opt_value, status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr);
opt_addr + opt_info->offset_);
} else { } else {
status = Status::InvalidArgument("Unrecognized option", opt_name); status = Status::InvalidArgument("Unrecognized option", opt_name);
} }
@ -1107,8 +1005,7 @@ Status OptionTypeInfo::ParseStruct(
std::string elem_name; std::string elem_name;
const auto opt_info = Find(opt_name, *struct_map, &elem_name); const auto opt_info = Find(opt_name, *struct_map, &elem_name);
if (opt_info != nullptr) { if (opt_info != nullptr) {
status = opt_info->Parse(config_options, elem_name, opt_value, status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr);
opt_addr + opt_info->offset_);
} else { } else {
status = Status::InvalidArgument("Unrecognized option", status = Status::InvalidArgument("Unrecognized option",
struct_name + "." + opt_name); struct_name + "." + opt_name);
@ -1119,18 +1016,30 @@ Status OptionTypeInfo::ParseStruct(
Status OptionTypeInfo::Serialize(const ConfigOptions& config_options, Status OptionTypeInfo::Serialize(const ConfigOptions& config_options,
const std::string& opt_name, const std::string& opt_name,
const char* opt_addr, const void* const opt_ptr,
std::string* opt_value) const { std::string* opt_value) const {
// If the option is no longer used in rocksdb and marked as deprecated, // If the option is no longer used in rocksdb and marked as deprecated,
// we skip it in the serialization. // we skip it in the serialization.
if (opt_addr != nullptr && ShouldSerialize()) { const char* opt_addr = reinterpret_cast<const char*>(opt_ptr) + offset_;
if (serialize_func_ != nullptr) { if (opt_addr == nullptr || IsDeprecated()) {
return Status::OK();
} else if (IsEnabled(OptionTypeFlags::kDontSerialize)) {
return Status::NotSupported("Cannot serialize option: ", opt_name);
} else if (serialize_func_ != nullptr) {
return serialize_func_(config_options, opt_name, opt_addr, opt_value); return serialize_func_(config_options, opt_name, opt_addr, opt_value);
} else if (!SerializeSingleOptionHelper(opt_addr, type_, opt_value)) { } else if (SerializeSingleOptionHelper(opt_addr, type_, opt_value)) {
return Status::InvalidArgument("Cannot serialize option", opt_name); return Status::OK();
} } else if (IsConfigurable()) {
const Configurable* config = AsRawPointer<Configurable>(opt_ptr);
if (config != nullptr) {
ConfigOptions embedded = config_options;
embedded.delimiter = ";";
*opt_value = config->ToString(embedded);
} }
return Status::OK(); return Status::OK();
} else {
return Status::InvalidArgument("Cannot serialize option: ", opt_name);
}
} }
Status OptionTypeInfo::SerializeStruct( Status OptionTypeInfo::SerializeStruct(
@ -1151,8 +1060,7 @@ Status OptionTypeInfo::SerializeStruct(
std::string single; std::string single;
const auto& opt_info = iter.second; const auto& opt_info = iter.second;
if (opt_info.ShouldSerialize()) { if (opt_info.ShouldSerialize()) {
status = opt_info.Serialize(embedded, iter.first, status = opt_info.Serialize(embedded, iter.first, opt_addr, &single);
opt_addr + opt_info.offset_, &single);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} else { } else {
@ -1167,8 +1075,7 @@ Status OptionTypeInfo::SerializeStruct(
const auto opt_info = const auto opt_info =
Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name); Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
if (opt_info != nullptr) { if (opt_info != nullptr) {
status = opt_info->Serialize(config_options, elem_name, status = opt_info->Serialize(config_options, elem_name, opt_addr, value);
opt_addr + opt_info->offset_, value);
} else { } else {
status = Status::InvalidArgument("Unrecognized option", opt_name); status = Status::InvalidArgument("Unrecognized option", opt_name);
} }
@ -1180,7 +1087,7 @@ Status OptionTypeInfo::SerializeStruct(
status = Status::InvalidArgument("Unrecognized option", opt_name); status = Status::InvalidArgument("Unrecognized option", opt_name);
} else if (opt_info->ShouldSerialize()) { } else if (opt_info->ShouldSerialize()) {
status = opt_info->Serialize(config_options, opt_name + "." + elem_name, status = opt_info->Serialize(config_options, opt_name + "." + elem_name,
opt_addr + opt_info->offset_, value); opt_addr, value);
} }
} }
return status; return status;
@ -1251,11 +1158,15 @@ static bool AreOptionsEqual(OptionType type, const char* this_offset,
bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options, bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
const std::string& opt_name, const std::string& opt_name,
const char* this_addr, const char* that_addr, const void* const this_ptr,
const void* const that_ptr,
std::string* mismatch) const { std::string* mismatch) const {
if (!config_options.IsCheckEnabled(GetSanityLevel())) { auto level = GetSanityLevel();
if (!config_options.IsCheckEnabled(level)) {
return true; // If the sanity level is not being checked, skip it return true; // If the sanity level is not being checked, skip it
} }
const auto this_addr = reinterpret_cast<const char*>(this_ptr) + offset_;
const auto that_addr = reinterpret_cast<const char*>(that_ptr) + offset_;
if (this_addr == nullptr || that_addr == nullptr) { if (this_addr == nullptr || that_addr == nullptr) {
if (this_addr == that_addr) { if (this_addr == that_addr) {
return true; return true;
@ -1267,6 +1178,27 @@ bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
} }
} else if (AreOptionsEqual(type_, this_addr, that_addr)) { } else if (AreOptionsEqual(type_, this_addr, that_addr)) {
return true; return true;
} else if (IsConfigurable()) {
const auto* this_config = AsRawPointer<Configurable>(this_ptr);
const auto* that_config = AsRawPointer<Configurable>(that_ptr);
if (this_config == that_config) {
return true;
} else if (this_config != nullptr && that_config != nullptr) {
std::string bad_name;
bool matches;
if (level < config_options.sanity_level) {
ConfigOptions copy = config_options;
copy.sanity_level = level;
matches = this_config->AreEquivalent(copy, that_config, &bad_name);
} else {
matches =
this_config->AreEquivalent(config_options, that_config, &bad_name);
}
if (!matches) {
*mismatch = opt_name + "." + bad_name;
}
return matches;
}
} }
if (mismatch->empty()) { if (mismatch->empty()) {
*mismatch = opt_name; *mismatch = opt_name;
@ -1287,9 +1219,8 @@ bool OptionTypeInfo::StructsAreEqual(
for (const auto& iter : *struct_map) { for (const auto& iter : *struct_map) {
const auto& opt_info = iter.second; const auto& opt_info = iter.second;
matches = opt_info.AreEqual(config_options, iter.first, matches = opt_info.AreEqual(config_options, iter.first, this_addr,
this_addr + opt_info.offset_, that_addr, &result);
that_addr + opt_info.offset_, &result);
if (!matches) { if (!matches) {
*mismatch = struct_name + "." + result; *mismatch = struct_name + "." + result;
return false; return false;
@ -1304,9 +1235,8 @@ bool OptionTypeInfo::StructsAreEqual(
if (opt_info == nullptr) { if (opt_info == nullptr) {
*mismatch = opt_name; *mismatch = opt_name;
matches = false; matches = false;
} else if (!opt_info->AreEqual(config_options, elem_name, } else if (!opt_info->AreEqual(config_options, elem_name, this_addr,
this_addr + opt_info->offset_, that_addr, &result)) {
that_addr + opt_info->offset_, &result)) {
matches = false; matches = false;
*mismatch = struct_name + "." + result; *mismatch = struct_name + "." + result;
} }
@ -1318,9 +1248,8 @@ bool OptionTypeInfo::StructsAreEqual(
if (opt_info == nullptr) { if (opt_info == nullptr) {
*mismatch = struct_name + "." + opt_name; *mismatch = struct_name + "." + opt_name;
matches = false; matches = false;
} else if (!opt_info->AreEqual(config_options, elem_name, } else if (!opt_info->AreEqual(config_options, elem_name, this_addr,
this_addr + opt_info->offset_, that_addr, &result)) {
that_addr + opt_info->offset_, &result)) {
matches = false; matches = false;
*mismatch = struct_name + "." + result; *mismatch = struct_name + "." + result;
} }
@ -1328,14 +1257,34 @@ bool OptionTypeInfo::StructsAreEqual(
return matches; return matches;
} }
bool MatchesOptionsTypeFromMap(
const ConfigOptions& config_options,
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
const void* const this_ptr, const void* const that_ptr,
std::string* mismatch) {
for (auto& pair : type_map) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) {
if (!pair.second.AreEqual(config_options, pair.first, this_ptr, that_ptr,
mismatch) &&
!pair.second.AreEqualByName(config_options, pair.first, this_ptr,
that_ptr)) {
return false;
}
}
}
return true;
}
bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options, bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
const std::string& opt_name, const std::string& opt_name,
const char* this_addr, const void* const this_ptr,
const char* that_addr) const { const void* const that_ptr) const {
if (IsByName()) { if (IsByName()) {
std::string that_value; std::string that_value;
if (Serialize(config_options, opt_name, that_addr, &that_value).ok()) { if (Serialize(config_options, opt_name, that_ptr, &that_value).ok()) {
return AreEqualByName(config_options, opt_name, this_addr, that_value); return AreEqualByName(config_options, opt_name, this_ptr, that_value);
} }
} }
return false; return false;
@ -1343,12 +1292,12 @@ bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options, bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
const std::string& opt_name, const std::string& opt_name,
const char* opt_addr, const void* const opt_ptr,
const std::string& that_value) const { const std::string& that_value) const {
std::string this_value; std::string this_value;
if (!IsByName()) { if (!IsByName()) {
return false; return false;
} else if (!Serialize(config_options, opt_name, opt_addr, &this_value).ok()) { } else if (!Serialize(config_options, opt_name, opt_ptr, &this_value).ok()) {
return false; return false;
} else if (IsEnabled(OptionVerificationType::kByNameAllowFromNull)) { } else if (IsEnabled(OptionVerificationType::kByNameAllowFromNull)) {
if (that_value == kNullptrString) { if (that_value == kNullptrString) {
@ -1376,7 +1325,8 @@ const OptionTypeInfo* OptionTypeInfo::Find(
auto siter = auto siter =
opt_map.find(opt_name.substr(0, idx)); // Look for the short name opt_map.find(opt_name.substr(0, idx)); // Look for the short name
if (siter != opt_map.end()) { // We found the short name if (siter != opt_map.end()) { // We found the short name
if (siter->second.IsStruct()) { // If the object is a struct if (siter->second.IsStruct() || // If the object is a struct
siter->second.IsConfigurable()) { // or a Configurable
*elem_name = opt_name.substr(idx + 1); // Return the rest *elem_name = opt_name.substr(idx + 1); // Return the rest
return &(siter->second); // Return the contents of the iterator return &(siter->second); // Return the contents of the iterator
} }

@ -10,19 +10,25 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "options/cf_options.h"
#include "options/db_options.h"
#include "options/options_type.h"
#include "rocksdb/options.h" #include "rocksdb/options.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "rocksdb/universal_compaction.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
struct ColumnFamilyOptions;
struct ConfigOptions; struct ConfigOptions;
struct DBOptions;
struct ImmutableDBOptions;
struct MutableDBOptions;
struct MutableCFOptions;
struct Options;
std::vector<CompressionType> GetSupportedCompressions(); std::vector<CompressionType> GetSupportedCompressions();
// Checks that the combination of DBOptions and ColumnFamilyOptions are valid
Status ValidateOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts);
DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options, DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
const MutableDBOptions& mutable_db_options); const MutableDBOptions& mutable_db_options);
@ -31,15 +37,22 @@ ColumnFamilyOptions BuildColumnFamilyOptions(
const MutableCFOptions& mutable_cf_options); const MutableCFOptions& mutable_cf_options);
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
Status GetStringFromStruct( std::unique_ptr<Configurable> DBOptionsAsConfigurable(
const ConfigOptions& config_options, const void* const opt_ptr, const MutableDBOptions& opts);
const std::unordered_map<std::string, OptionTypeInfo>& type_info, std::unique_ptr<Configurable> DBOptionsAsConfigurable(const DBOptions& opts);
std::unique_ptr<Configurable> CFOptionsAsConfigurable(
const MutableCFOptions& opts);
std::unique_ptr<Configurable> CFOptionsAsConfigurable(
const ColumnFamilyOptions& opts,
const std::unordered_map<std::string, std::string>* opt_map = nullptr);
Status GetStringFromMutableCFOptions(const ConfigOptions& config_options,
const MutableCFOptions& mutable_opts,
std::string* opt_string); std::string* opt_string);
Status ParseColumnFamilyOption(const ConfigOptions& config_options, Status GetStringFromMutableDBOptions(const ConfigOptions& config_options,
const std::string& name, const MutableDBOptions& mutable_opts,
const std::string& org_value, std::string* opt_string);
ColumnFamilyOptions* new_options);
Status GetMutableOptionsFromStrings( Status GetMutableOptionsFromStrings(
const MutableCFOptions& base_options, const MutableCFOptions& base_options,
@ -51,31 +64,6 @@ Status GetMutableDBOptionsFromStrings(
const std::unordered_map<std::string, std::string>& options_map, const std::unordered_map<std::string, std::string>& options_map,
MutableDBOptions* new_options); MutableDBOptions* new_options);
Status GetTableFactoryFromMap(
const std::string& factory_name,
const std::unordered_map<std::string, std::string>& opt_map,
std::shared_ptr<TableFactory>* table_factory,
bool ignore_unknown_options = false);
Status GetTableFactoryFromMap(
const ConfigOptions& config_options, const std::string& factory_name,
const std::unordered_map<std::string, std::string>& opt_map,
std::shared_ptr<TableFactory>* table_factory);
// A helper function that converts "opt_address" to a std::string
// based on the specified OptionType.
bool SerializeSingleOptionHelper(const char* opt_address,
const OptionType opt_type, std::string* value);
// In addition to its public version defined in rocksdb/convenience.h,
// this further takes an optional output vector "unsupported_options_names",
// which stores the name of all the unsupported options specified in "opts_map".
Status GetDBOptionsFromMapInternal(
const ConfigOptions& config_options, const DBOptions& base_options,
const std::unordered_map<std::string, std::string>& opts_map,
DBOptions* new_options,
std::vector<std::string>* unsupported_options_names = nullptr);
bool ParseSliceTransform( bool ParseSliceTransform(
const std::string& value, const std::string& value,
std::shared_ptr<const SliceTransform>* slice_transform); std::shared_ptr<const SliceTransform>* slice_transform);
@ -83,12 +71,11 @@ bool ParseSliceTransform(
extern Status StringToMap( extern Status StringToMap(
const std::string& opts_str, const std::string& opts_str,
std::unordered_map<std::string, std::string>* opts_map); std::unordered_map<std::string, std::string>* opts_map);
extern bool ParseOptionHelper(char* opt_address, const OptionType& opt_type,
const std::string& value);
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
struct OptionsHelper { struct OptionsHelper {
static const std::string kCFOptionsName /*= "ColumnFamilyOptions"*/;
static const std::string kDBOptionsName /*= "DBOptions" */;
static std::map<CompactionStyle, std::string> compaction_style_to_string; static std::map<CompactionStyle, std::string> compaction_style_to_string;
static std::map<CompactionPri, std::string> compaction_pri_to_string; static std::map<CompactionPri, std::string> compaction_pri_to_string;
static std::map<CompactionStopStyle, std::string> static std::map<CompactionStopStyle, std::string>
@ -97,16 +84,13 @@ struct OptionsHelper {
static std::unordered_map<std::string, CompressionType> static std::unordered_map<std::string, CompressionType>
compression_type_string_map; compression_type_string_map;
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info;
static std::unordered_map<std::string, CompactionStopStyle> static std::unordered_map<std::string, CompactionStopStyle>
compaction_stop_style_string_map; compaction_stop_style_string_map;
static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info;
static std::unordered_map<std::string, EncodingType> encoding_type_string_map; static std::unordered_map<std::string, EncodingType> encoding_type_string_map;
static std::unordered_map<std::string, CompactionStyle> static std::unordered_map<std::string, CompactionStyle>
compaction_style_string_map; compaction_style_string_map;
static std::unordered_map<std::string, CompactionPri> static std::unordered_map<std::string, CompactionPri>
compaction_pri_string_map; compaction_pri_string_map;
static ColumnFamilyOptions dummy_cf_options;
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
}; };
@ -118,10 +102,8 @@ static auto& compaction_stop_style_to_string =
OptionsHelper::compaction_stop_style_to_string; OptionsHelper::compaction_stop_style_to_string;
static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map; static auto& checksum_type_string_map = OptionsHelper::checksum_type_string_map;
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
static auto& cf_options_type_info = OptionsHelper::cf_options_type_info;
static auto& compaction_stop_style_string_map = static auto& compaction_stop_style_string_map =
OptionsHelper::compaction_stop_style_string_map; OptionsHelper::compaction_stop_style_string_map;
static auto& db_options_type_info = OptionsHelper::db_options_type_info;
static auto& compression_type_string_map = static auto& compression_type_string_map =
OptionsHelper::compression_type_string_map; OptionsHelper::compression_type_string_map;
static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map; static auto& encoding_type_string_map = OptionsHelper::encoding_type_string_map;

@ -15,11 +15,13 @@
#include "file/read_write_util.h" #include "file/read_write_util.h"
#include "file/writable_file_writer.h" #include "file/writable_file_writer.h"
#include "options/cf_options.h"
#include "options/db_options.h"
#include "options/options_helper.h" #include "options/options_helper.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "table/block_based/block_based_table_factory.h" #include "rocksdb/utilities/options_type.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "util/cast_util.h" #include "util/cast_util.h"
#include "util/string_util.h" #include "util/string_util.h"
@ -41,6 +43,8 @@ Status PersistRocksDBOptions(const DBOptions& db_opt,
ConfigOptions ConfigOptions
config_options; // Use default for escaped(true) and check (exact) config_options; // Use default for escaped(true) and check (exact)
config_options.delimiter = "\n "; config_options.delimiter = "\n ";
// Do not invoke PrepareOptions when we are doing validation.
config_options.invoke_prepare_options = false;
// If a readahead size was set in the input options, use it // If a readahead size was set in the input options, use it
if (db_opt.log_readahead_size > 0) { if (db_opt.log_readahead_size > 0) {
config_options.file_readahead_size = db_opt.log_readahead_size; config_options.file_readahead_size = db_opt.log_readahead_size;
@ -450,13 +454,18 @@ Status RocksDBOptionsParser::EndSection(
section_arg); section_arg);
} }
// Ignore error as table factory deserialization is optional // Ignore error as table factory deserialization is optional
s = GetTableFactoryFromMap( s = TableFactory::CreateFromString(
config_options, config_options,
section_title.substr( section_title.substr(
opt_section_titles[kOptionSectionTableOptions].size()), opt_section_titles[kOptionSectionTableOptions].size()),
opt_map, &(cf_opt->table_factory)); &(cf_opt->table_factory));
if (!s.ok()) { if (s.ok()) {
return s; return cf_opt->table_factory->ConfigureFromMap(config_options, opt_map);
} else {
// Return OK for not supported table factories as TableFactory
// Deserialization is optional.
cf_opt->table_factory.reset();
return Status::OK();
} }
} else if (section == kOptionSectionVersion) { } else if (section == kOptionSectionVersion) {
for (const auto& pair : opt_map) { for (const auto& pair : opt_map) {
@ -531,11 +540,14 @@ std::string RocksDBOptionsParser::TrimAndRemoveComment(const std::string& line,
} }
Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
const ConfigOptions& config_options, const DBOptions& db_opt, const ConfigOptions& config_options_in, const DBOptions& db_opt,
const std::vector<std::string>& cf_names, const std::vector<std::string>& cf_names,
const std::vector<ColumnFamilyOptions>& cf_opts, const std::vector<ColumnFamilyOptions>& cf_opts,
const std::string& file_name, FileSystem* fs) { const std::string& file_name, FileSystem* fs) {
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ConfigOptions config_options = config_options_in;
config_options.invoke_prepare_options =
false; // No need to do a prepare for verify
Status s = parser.Parse(config_options, file_name, fs); Status s = parser.Parse(config_options, file_name, fs);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
@ -606,55 +618,36 @@ Status RocksDBOptionsParser::VerifyDBOptions(
const ConfigOptions& config_options, const DBOptions& base_opt, const ConfigOptions& config_options, const DBOptions& base_opt,
const DBOptions& file_opt, const DBOptions& file_opt,
const std::unordered_map<std::string, std::string>* /*opt_map*/) { const std::unordered_map<std::string, std::string>* /*opt_map*/) {
for (const auto& pair : db_options_type_info) { auto base_config = DBOptionsAsConfigurable(base_opt);
const auto& opt_info = pair.second; auto file_config = DBOptionsAsConfigurable(file_opt);
if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) {
const char* base_addr =
reinterpret_cast<const char*>(&base_opt) + opt_info.offset_;
const char* file_addr =
reinterpret_cast<const char*>(&file_opt) + opt_info.offset_;
std::string mismatch; std::string mismatch;
if (!opt_info.AreEqual(config_options, pair.first, base_addr, file_addr, if (!base_config->AreEquivalent(config_options, file_config.get(),
&mismatch) && &mismatch)) {
!opt_info.AreEqualByName(config_options, pair.first, base_addr,
file_addr)) {
const size_t kBufferSize = 2048; const size_t kBufferSize = 2048;
char buffer[kBufferSize]; char buffer[kBufferSize];
std::string base_value; std::string base_value;
std::string file_value; std::string file_value;
int offset = int offset = snprintf(buffer, sizeof(buffer),
snprintf(buffer, sizeof(buffer),
"[RocksDBOptionsParser]: " "[RocksDBOptionsParser]: "
"failed the verification on ColumnFamilyOptions::%s", "failed the verification on DBOptions::%s -- ",
pair.first.c_str()); mismatch.c_str());
Status s = opt_info.Serialize(config_options, pair.first, base_addr, Status s = base_config->GetOption(config_options, mismatch, &base_value);
&base_value);
if (s.ok()) { if (s.ok()) {
s = opt_info.Serialize(config_options, pair.first, file_addr, s = file_config->GetOption(config_options, mismatch, &file_value);
&file_value);
} }
snprintf(buffer, sizeof(buffer),
"[RocksDBOptionsParser]: "
"failed the verification on DBOptions::%s --- "
"The specified one is %s while the persisted one is %s.\n",
pair.first.c_str(), base_value.c_str(), file_value.c_str());
assert(offset >= 0); assert(offset >= 0);
assert(static_cast<size_t>(offset) < sizeof(buffer)); assert(static_cast<size_t>(offset) < sizeof(buffer));
if (s.ok()) { if (s.ok()) {
snprintf( snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
buffer + offset, sizeof(buffer) - static_cast<size_t>(offset), "-- The specified one is %s while the persisted one is %s.\n",
"--- The specified one is %s while the persisted one is %s.\n",
base_value.c_str(), file_value.c_str()); base_value.c_str(), file_value.c_str());
} else { } else {
snprintf(buffer + offset, snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
sizeof(buffer) - static_cast<size_t>(offset), "-- Unable to re-serialize an option: %s.\n",
"--- Unable to re-serialize an option: %s.\n",
s.ToString().c_str()); s.ToString().c_str());
} }
return Status::InvalidArgument(Slice(buffer, strlen(buffer))); return Status::InvalidArgument(Slice(buffer, strlen(buffer)));
} }
}
}
return Status::OK(); return Status::OK();
} }
@ -662,63 +655,36 @@ Status RocksDBOptionsParser::VerifyCFOptions(
const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt, const ConfigOptions& config_options, const ColumnFamilyOptions& base_opt,
const ColumnFamilyOptions& file_opt, const ColumnFamilyOptions& file_opt,
const std::unordered_map<std::string, std::string>* opt_map) { const std::unordered_map<std::string, std::string>* opt_map) {
for (const auto& pair : cf_options_type_info) { auto base_config = CFOptionsAsConfigurable(base_opt, opt_map);
const auto& opt_info = pair.second; auto file_config = CFOptionsAsConfigurable(file_opt, opt_map);
if (config_options.IsCheckEnabled(opt_info.GetSanityLevel())) {
std::string mismatch; std::string mismatch;
const char* base_addr = if (!base_config->AreEquivalent(config_options, file_config.get(),
reinterpret_cast<const char*>(&base_opt) + opt_info.offset_; &mismatch)) {
const char* file_addr = std::string base_value;
reinterpret_cast<const char*>(&file_opt) + opt_info.offset_; std::string file_value;
bool matches = opt_info.AreEqual(config_options, pair.first, base_addr,
file_addr, &mismatch);
if (!matches && opt_info.IsByName()) {
if (opt_map == nullptr) {
matches = true;
} else {
auto iter = opt_map->find(pair.first);
if (iter == opt_map->end()) {
matches = true;
} else {
matches = opt_info.AreEqualByName(config_options, pair.first,
base_addr, iter->second);
}
}
}
if (!matches) {
// The options do not match // The options do not match
const size_t kBufferSize = 2048; const size_t kBufferSize = 2048;
char buffer[kBufferSize]; char buffer[kBufferSize];
std::string base_value; Status s = base_config->GetOption(config_options, mismatch, &base_value);
std::string file_value;
Status s = opt_info.Serialize(config_options, pair.first, base_addr,
&base_value);
if (s.ok()) { if (s.ok()) {
s = opt_info.Serialize(config_options, pair.first, file_addr, s = file_config->GetOption(config_options, mismatch, &file_value);
&file_value);
} }
int offset = int offset = snprintf(buffer, sizeof(buffer),
snprintf(buffer, sizeof(buffer),
"[RocksDBOptionsParser]: " "[RocksDBOptionsParser]: "
"failed the verification on ColumnFamilyOptions::%s", "failed the verification on ColumnFamilyOptions::%s",
pair.first.c_str()); mismatch.c_str());
assert(offset >= 0); assert(offset >= 0);
assert(static_cast<size_t>(offset) < sizeof(buffer)); assert(static_cast<size_t>(offset) < sizeof(buffer));
if (s.ok()) { if (s.ok()) {
snprintf( snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
"--- The specified one is %s while the persisted one is %s.\n", "--- The specified one is %s while the persisted one is %s.\n",
base_value.c_str(), file_value.c_str()); base_value.c_str(), file_value.c_str());
} else { } else {
snprintf(buffer + offset, snprintf(buffer + offset, sizeof(buffer) - static_cast<size_t>(offset),
sizeof(buffer) - static_cast<size_t>(offset),
"--- Unable to re-serialize an option: %s.\n", "--- Unable to re-serialize an option: %s.\n",
s.ToString().c_str()); s.ToString().c_str());
} }
return Status::InvalidArgument(Slice(buffer, sizeof(buffer))); return Status::InvalidArgument(Slice(buffer, sizeof(buffer)));
} // if (! matches)
} // CheckSanityLevel
} // For each option } // For each option
return Status::OK(); return Status::OK();
} }
@ -726,22 +692,19 @@ Status RocksDBOptionsParser::VerifyCFOptions(
Status RocksDBOptionsParser::VerifyTableFactory( Status RocksDBOptionsParser::VerifyTableFactory(
const ConfigOptions& config_options, const TableFactory* base_tf, const ConfigOptions& config_options, const TableFactory* base_tf,
const TableFactory* file_tf) { const TableFactory* file_tf) {
std::string mismatch;
if (base_tf && file_tf) { if (base_tf && file_tf) {
if (config_options.sanity_level > ConfigOptions::kSanityLevelNone && if (config_options.sanity_level > ConfigOptions::kSanityLevelNone &&
std::string(base_tf->Name()) != std::string(file_tf->Name())) { std::string(base_tf->Name()) != std::string(file_tf->Name())) {
return Status::Corruption( return Status::Corruption(
"[RocksDBOptionsParser]: " "[RocksDBOptionsParser]: "
"failed the verification on TableFactory->Name()"); "failed the verification on TableFactory->Name()");
} else if (!base_tf->AreEquivalent(config_options, file_tf, &mismatch)) {
return Status::Corruption(std::string("[RocksDBOptionsParser]:"
"failed the verification on ") +
base_tf->Name() + "::",
mismatch);
} }
if (base_tf->Name() == BlockBasedTableFactory::kName) {
return VerifyBlockBasedTableFactory(
config_options,
static_cast_with_check<const BlockBasedTableFactory,
const TableFactory>(base_tf),
static_cast_with_check<const BlockBasedTableFactory,
const TableFactory>(file_tf));
}
// TODO(yhchiang): add checks for other table factory types
} else { } else {
// TODO(yhchiang): further support sanity check here // TODO(yhchiang): further support sanity check here
} }

@ -98,6 +98,9 @@ class RocksDBOptionsParser {
static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser); static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser);
static Status ParseStatement(std::string* name, std::string* value,
const std::string& line, const int line_num);
protected: protected:
bool IsSection(const std::string& line); bool IsSection(const std::string& line);
Status ParseSection(OptionSection* section, std::string* title, Status ParseSection(OptionSection* section, std::string* title,
@ -107,9 +110,6 @@ class RocksDBOptionsParser {
Status CheckSection(const OptionSection section, Status CheckSection(const OptionSection section,
const std::string& section_arg, const int line_num); const std::string& section_arg, const int line_num);
Status ParseStatement(std::string* name, std::string* value,
const std::string& line, const int line_num);
Status EndSection( Status EndSection(
const ConfigOptions& config_options, const OptionSection section, const ConfigOptions& config_options, const OptionSection section,
const std::string& title, const std::string& section_arg, const std::string& title, const std::string& section_arg,
@ -117,7 +117,7 @@ class RocksDBOptionsParser {
Status ValidityCheck(); Status ValidityCheck();
Status InvalidArgument(const int line_num, const std::string& message); static Status InvalidArgument(const int line_num, const std::string& message);
Status ParseVersionNumber(const std::string& ver_name, Status ParseVersionNumber(const std::string& ver_name,
const std::string& ver_string, const int max_count, const std::string& ver_string, const int max_count,

@ -9,6 +9,8 @@
#include <cstring> #include <cstring>
#include "options/cf_options.h"
#include "options/db_options.h"
#include "options/options_helper.h" #include "options/options_helper.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "test_util/testharness.h" #include "test_util/testharness.h"
@ -424,6 +426,7 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
options->compaction_options_universal = CompactionOptionsUniversal(); options->compaction_options_universal = CompactionOptionsUniversal();
options->hard_rate_limit = 0; options->hard_rate_limit = 0;
options->soft_rate_limit = 0; options->soft_rate_limit = 0;
options->num_levels = 42; // Initialize options for MutableCF
options->purge_redundant_kvs_while_flush = false; options->purge_redundant_kvs_while_flush = false;
options->max_mem_compaction_level = 0; options->max_mem_compaction_level = 0;
options->compaction_filter = nullptr; options->compaction_filter = nullptr;

@ -22,6 +22,7 @@
#include "rocksdb/memtablerep.h" #include "rocksdb/memtablerep.h"
#include "rocksdb/utilities/leveldb_options.h" #include "rocksdb/utilities/leveldb_options.h"
#include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "table/block_based/filter_policy_internal.h" #include "table/block_based/filter_policy_internal.h"
#include "test_util/testharness.h" #include "test_util/testharness.h"
#include "test_util/testutil.h" #include "test_util/testutil.h"
@ -817,7 +818,7 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
ASSERT_OK(GetBlockBasedTableOptionsFromString( ASSERT_OK(GetBlockBasedTableOptionsFromString(
config_options, table_opt, config_options, table_opt,
"cache_index_and_filter_blocks=1;index_type=kHashSearch;" "cache_index_and_filter_blocks=1;index_type=kHashSearch;"
"checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" "checksum=kxxHash;hash_index_allow_collision=1;"
"block_cache=1M;block_cache_compressed=1k;block_size=1024;" "block_cache=1M;block_cache_compressed=1k;block_size=1024;"
"block_size_deviation=8;block_restart_interval=4;" "block_size_deviation=8;block_restart_interval=4;"
"format_version=5;whole_key_filtering=1;" "format_version=5;whole_key_filtering=1;"
@ -827,7 +828,6 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch); ASSERT_EQ(new_opt.index_type, BlockBasedTableOptions::kHashSearch);
ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash); ASSERT_EQ(new_opt.checksum, ChecksumType::kxxHash);
ASSERT_TRUE(new_opt.hash_index_allow_collision); ASSERT_TRUE(new_opt.hash_index_allow_collision);
ASSERT_TRUE(new_opt.no_block_cache);
ASSERT_TRUE(new_opt.block_cache != nullptr); ASSERT_TRUE(new_opt.block_cache != nullptr);
ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL); ASSERT_EQ(new_opt.block_cache->GetCapacity(), 1024UL*1024UL);
ASSERT_TRUE(new_opt.block_cache_compressed != nullptr); ASSERT_TRUE(new_opt.block_cache_compressed != nullptr);
@ -1127,13 +1127,13 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) {
ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false); ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false);
ASSERT_EQ(new_options.write_buffer_size, 10U); ASSERT_EQ(new_options.write_buffer_size, 10U);
ASSERT_EQ(new_options.max_write_buffer_number, 16); ASSERT_EQ(new_options.max_write_buffer_number, 16);
BlockBasedTableOptions new_block_based_table_options = const auto new_bbto =
dynamic_cast<BlockBasedTableFactory*>(new_options.table_factory.get()) new_options.table_factory->GetOptions<BlockBasedTableOptions>();
->table_options(); ASSERT_NE(new_bbto, nullptr);
ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20); ASSERT_EQ(new_bbto->block_cache->GetCapacity(), 1U << 20);
ASSERT_EQ(new_block_based_table_options.block_size, 4U); ASSERT_EQ(new_bbto->block_size, 4U);
// don't overwrite block based table options // don't overwrite block based table options
ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks); ASSERT_TRUE(new_bbto->cache_index_and_filter_blocks);
ASSERT_EQ(new_options.create_if_missing, true); ASSERT_EQ(new_options.create_if_missing, true);
ASSERT_EQ(new_options.max_open_files, 1); ASSERT_EQ(new_options.max_open_files, 1);
@ -1142,7 +1142,42 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) {
ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv)); ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv));
ASSERT_EQ(newEnv, new_options.env); ASSERT_EQ(newEnv, new_options.env);
// Test the old interfaxe config_options.ignore_unknown_options = false;
// Test a bad value for a DBOption returns a failure
base_options.dump_malloc_stats = false;
base_options.write_buffer_size = 1024;
Options bad_options = new_options;
ASSERT_NOK(GetOptionsFromString(config_options, base_options,
"create_if_missing=XX;dump_malloc_stats=true",
&bad_options));
ASSERT_EQ(bad_options.dump_malloc_stats, false);
bad_options = new_options;
ASSERT_NOK(GetOptionsFromString(config_options, base_options,
"write_buffer_size=XX;dump_malloc_stats=true",
&bad_options));
ASSERT_EQ(bad_options.dump_malloc_stats, false);
// Test a bad value for a TableFactory Option returns a failure
bad_options = new_options;
ASSERT_NOK(GetOptionsFromString(config_options, base_options,
"write_buffer_size=16;dump_malloc_stats=true"
"block_based_table_factory={block_size=XX;};",
&bad_options));
ASSERT_EQ(bad_options.dump_malloc_stats, false);
ASSERT_EQ(bad_options.write_buffer_size, 1024);
config_options.ignore_unknown_options = true;
ASSERT_OK(GetOptionsFromString(config_options, base_options,
"create_if_missing=XX;dump_malloc_stats=true;"
"write_buffer_size=XX;"
"block_based_table_factory={block_size=XX;};",
&bad_options));
ASSERT_EQ(bad_options.create_if_missing, base_options.create_if_missing);
ASSERT_EQ(bad_options.dump_malloc_stats, true);
ASSERT_EQ(bad_options.write_buffer_size, base_options.write_buffer_size);
// Test the old interface
ASSERT_OK(GetOptionsFromString( ASSERT_OK(GetOptionsFromString(
base_options, base_options,
"write_buffer_size=22;max_write_buffer_number=33;max_open_files=44;", "write_buffer_size=22;max_write_buffer_number=33;max_open_files=44;",
@ -1227,6 +1262,76 @@ TEST_F(OptionsTest, ColumnFamilyOptionsSerialization) {
} }
} }
TEST_F(OptionsTest, CheckBlockBasedTableOptions) {
ColumnFamilyOptions cf_opts;
DBOptions db_opts;
ConfigOptions config_opts;
ASSERT_OK(GetColumnFamilyOptionsFromString(
config_opts, cf_opts, "prefix_extractor=capped:8", &cf_opts));
ASSERT_OK(TableFactory::CreateFromString(config_opts, "BlockBasedTable",
&cf_opts.table_factory));
ASSERT_NE(cf_opts.table_factory.get(), nullptr);
ASSERT_TRUE(cf_opts.table_factory->IsInstanceOf(
TableFactory::kBlockBasedTableName()));
auto bbto = cf_opts.table_factory->GetOptions<BlockBasedTableOptions>();
ASSERT_OK(cf_opts.table_factory->ConfigureFromString(
config_opts,
"block_cache={capacity=1M;num_shard_bits=4;};"
"block_size_deviation=101;"
"block_restart_interval=0;"
"index_block_restart_interval=5;"
"partition_filters=true;"
"index_type=kHashSearch;"
"no_block_cache=1;"));
ASSERT_NE(bbto, nullptr);
ASSERT_EQ(bbto->block_cache.get(), nullptr);
ASSERT_EQ(bbto->block_size_deviation, 0);
ASSERT_EQ(bbto->block_restart_interval, 1);
ASSERT_EQ(bbto->index_block_restart_interval, 1);
ASSERT_FALSE(bbto->partition_filters);
ASSERT_OK(TableFactory::CreateFromString(config_opts, "BlockBasedTable",
&cf_opts.table_factory));
bbto = cf_opts.table_factory->GetOptions<BlockBasedTableOptions>();
ASSERT_OK(cf_opts.table_factory->ConfigureFromString(config_opts,
"no_block_cache=0;"));
ASSERT_NE(bbto->block_cache.get(), nullptr);
ASSERT_OK(cf_opts.table_factory->ValidateOptions(db_opts, cf_opts));
}
TEST_F(OptionsTest, MutableTableOptions) {
ConfigOptions config_options;
std::shared_ptr<TableFactory> bbtf;
bbtf.reset(NewBlockBasedTableFactory());
auto bbto = bbtf->GetOptions<BlockBasedTableOptions>();
ASSERT_NE(bbto, nullptr);
ASSERT_FALSE(bbtf->IsPrepared());
ASSERT_OK(bbtf->ConfigureOption(config_options, "block_align", "true"));
ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024"));
ASSERT_EQ(bbto->block_align, true);
ASSERT_EQ(bbto->block_size, 1024);
ASSERT_OK(bbtf->PrepareOptions(config_options));
ASSERT_TRUE(bbtf->IsPrepared());
ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "1024"));
ASSERT_EQ(bbto->block_align, true);
ASSERT_NOK(bbtf->ConfigureOption(config_options, "block_align", "false"));
ASSERT_OK(bbtf->ConfigureOption(config_options, "block_size", "2048"));
ASSERT_EQ(bbto->block_align, true);
ASSERT_EQ(bbto->block_size, 2048);
ColumnFamilyOptions cf_opts;
cf_opts.table_factory = bbtf;
ASSERT_NOK(GetColumnFamilyOptionsFromString(
config_options, cf_opts, "block_based_table_factory.block_align=false",
&cf_opts));
ASSERT_OK(GetColumnFamilyOptionsFromString(
config_options, cf_opts, "block_based_table_factory.block_size=8192",
&cf_opts));
ASSERT_EQ(bbto->block_align, true);
ASSERT_EQ(bbto->block_size, 8192);
}
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
Status StringToMap( Status StringToMap(
@ -1434,19 +1539,15 @@ TEST_F(OptionsTest, ConvertOptionsTest) {
ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files); ASSERT_EQ(converted_opt.max_open_files, leveldb_opt.max_open_files);
ASSERT_EQ(converted_opt.compression, leveldb_opt.compression); ASSERT_EQ(converted_opt.compression, leveldb_opt.compression);
std::shared_ptr<TableFactory> tb_guard = converted_opt.table_factory; std::shared_ptr<TableFactory> table_factory = converted_opt.table_factory;
BlockBasedTableFactory* table_factory = const auto table_opt = table_factory->GetOptions<BlockBasedTableOptions>();
dynamic_cast<BlockBasedTableFactory*>(converted_opt.table_factory.get()); ASSERT_NE(table_opt, nullptr);
ASSERT_TRUE(table_factory != nullptr);
const BlockBasedTableOptions table_opt = table_factory->table_options(); ASSERT_EQ(table_opt->block_cache->GetCapacity(), 8UL << 20);
ASSERT_EQ(table_opt->block_size, leveldb_opt.block_size);
ASSERT_EQ(table_opt.block_cache->GetCapacity(), 8UL << 20); ASSERT_EQ(table_opt->block_restart_interval,
ASSERT_EQ(table_opt.block_size, leveldb_opt.block_size);
ASSERT_EQ(table_opt.block_restart_interval,
leveldb_opt.block_restart_interval); leveldb_opt.block_restart_interval);
ASSERT_EQ(table_opt.filter_policy.get(), leveldb_opt.filter_policy); ASSERT_EQ(table_opt->filter_policy.get(), leveldb_opt.filter_policy);
} }
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
@ -2184,13 +2285,15 @@ TEST_F(OptionsOldApiTest, GetOptionsFromStringTest) {
ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false); ASSERT_EQ(new_options.bottommost_compression_opts.enabled, false);
ASSERT_EQ(new_options.write_buffer_size, 10U); ASSERT_EQ(new_options.write_buffer_size, 10U);
ASSERT_EQ(new_options.max_write_buffer_number, 16); ASSERT_EQ(new_options.max_write_buffer_number, 16);
BlockBasedTableOptions new_block_based_table_options =
dynamic_cast<BlockBasedTableFactory*>(new_options.table_factory.get()) auto new_block_based_table_options =
->table_options(); new_options.table_factory->GetOptions<BlockBasedTableOptions>();
ASSERT_EQ(new_block_based_table_options.block_cache->GetCapacity(), 1U << 20); ASSERT_NE(new_block_based_table_options, nullptr);
ASSERT_EQ(new_block_based_table_options.block_size, 4U); ASSERT_EQ(new_block_based_table_options->block_cache->GetCapacity(),
1U << 20);
ASSERT_EQ(new_block_based_table_options->block_size, 4U);
// don't overwrite block based table options // don't overwrite block based table options
ASSERT_TRUE(new_block_based_table_options.cache_index_and_filter_blocks); ASSERT_TRUE(new_block_based_table_options->cache_index_and_filter_blocks);
ASSERT_EQ(new_options.create_if_missing, true); ASSERT_EQ(new_options.create_if_missing, true);
ASSERT_EQ(new_options.max_open_files, 1); ASSERT_EQ(new_options.max_open_files, 1);
@ -2763,12 +2866,12 @@ TEST_F(OptionsParserTest, DumpAndParse) {
// Make sure block-based table factory options was deserialized correctly // Make sure block-based table factory options was deserialized correctly
std::shared_ptr<TableFactory> ttf = (*parser.cf_opts())[4].table_factory; std::shared_ptr<TableFactory> ttf = (*parser.cf_opts())[4].table_factory;
ASSERT_EQ(BlockBasedTableFactory::kName, std::string(ttf->Name())); ASSERT_EQ(TableFactory::kBlockBasedTableName(), std::string(ttf->Name()));
const BlockBasedTableOptions& parsed_bbto = const auto parsed_bbto = ttf->GetOptions<BlockBasedTableOptions>();
static_cast<BlockBasedTableFactory*>(ttf.get())->table_options(); ASSERT_NE(parsed_bbto, nullptr);
ASSERT_EQ(special_bbto.block_size, parsed_bbto.block_size); ASSERT_EQ(special_bbto.block_size, parsed_bbto->block_size);
ASSERT_EQ(special_bbto.cache_index_and_filter_blocks, ASSERT_EQ(special_bbto.cache_index_and_filter_blocks,
parsed_bbto.cache_index_and_filter_blocks); parsed_bbto->cache_index_and_filter_blocks);
ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( ASSERT_OK(RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName, config_options, base_db_opt, cf_names, base_cf_opts, kOptionsFileName,
@ -3181,11 +3284,9 @@ static void TestAndCompareOption(const ConfigOptions& config_options,
const std::string& opt_name, void* base_ptr, const std::string& opt_name, void* base_ptr,
void* comp_ptr) { void* comp_ptr) {
std::string result, mismatch; std::string result, mismatch;
char* base_addr = reinterpret_cast<char*>(base_ptr) + opt_info.offset_; ASSERT_OK(opt_info.Serialize(config_options, opt_name, base_ptr, &result));
char* comp_addr = reinterpret_cast<char*>(comp_ptr) + opt_info.offset_; ASSERT_OK(opt_info.Parse(config_options, opt_name, result, comp_ptr));
ASSERT_OK(opt_info.Serialize(config_options, opt_name, base_addr, &result)); ASSERT_TRUE(opt_info.AreEqual(config_options, opt_name, base_ptr, comp_ptr,
ASSERT_OK(opt_info.Parse(config_options, opt_name, result, comp_addr));
ASSERT_TRUE(opt_info.AreEqual(config_options, opt_name, base_addr, comp_addr,
&mismatch)); &mismatch));
} }
@ -3194,8 +3295,7 @@ static void TestAndCompareOption(const ConfigOptions& config_options,
const std::string& opt_name, const std::string& opt_name,
const std::string& opt_value, void* base_ptr, const std::string& opt_value, void* base_ptr,
void* comp_ptr) { void* comp_ptr) {
char* base_addr = reinterpret_cast<char*>(base_ptr) + opt_info.offset_; ASSERT_OK(opt_info.Parse(config_options, opt_name, opt_value, base_ptr));
ASSERT_OK(opt_info.Parse(config_options, opt_name, opt_value, base_addr));
TestAndCompareOption(config_options, opt_info, opt_name, base_ptr, comp_ptr); TestAndCompareOption(config_options, opt_info, opt_name, base_ptr, comp_ptr);
} }
@ -3204,13 +3304,10 @@ void TestOptInfo(const ConfigOptions& config_options, OptionType opt_type,
T* base, T* comp) { T* base, T* comp) {
std::string result; std::string result;
OptionTypeInfo opt_info(0, opt_type); OptionTypeInfo opt_info(0, opt_type);
char* base_addr = reinterpret_cast<char*>(base); ASSERT_FALSE(opt_info.AreEqual(config_options, "base", base, comp, &result));
char* comp_addr = reinterpret_cast<char*>(comp);
ASSERT_FALSE(
opt_info.AreEqual(config_options, "base", base_addr, comp_addr, &result));
ASSERT_EQ(result, "base"); ASSERT_EQ(result, "base");
ASSERT_NE(*base, *comp); ASSERT_NE(*base, *comp);
TestAndCompareOption(config_options, opt_info, "base", base_addr, comp_addr); TestAndCompareOption(config_options, opt_info, "base", base, comp);
ASSERT_EQ(*base, *comp); ASSERT_EQ(*base, *comp);
} }
@ -3273,53 +3370,46 @@ TEST_F(OptionTypeInfoTest, TestInvalidArgs) {
double d; double d;
ASSERT_NOK(OptionTypeInfo(0, OptionType::kBoolean) ASSERT_NOK(OptionTypeInfo(0, OptionType::kBoolean)
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&b))); .Parse(config_options, "b", "x", &b));
ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt)
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&i)));
ASSERT_NOK(
OptionTypeInfo(0, OptionType::kInt32T)
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&i32)));
ASSERT_NOK(
OptionTypeInfo(0, OptionType::kInt64T)
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&i64)));
ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt)
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&u)));
ASSERT_NOK(
OptionTypeInfo(0, OptionType::kUInt32T)
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&u32)));
ASSERT_NOK( ASSERT_NOK(
OptionTypeInfo(0, OptionType::kUInt64T) OptionTypeInfo(0, OptionType::kInt).Parse(config_options, "b", "x", &i));
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&u64))); ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt32T)
.Parse(config_options, "b", "x", &i32));
ASSERT_NOK(OptionTypeInfo(0, OptionType::kInt64T)
.Parse(config_options, "b", "x", &i64));
ASSERT_NOK( ASSERT_NOK(
OptionTypeInfo(0, OptionType::kSizeT) OptionTypeInfo(0, OptionType::kUInt).Parse(config_options, "b", "x", &u));
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&sz))); ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt32T)
.Parse(config_options, "b", "x", &u32));
ASSERT_NOK(OptionTypeInfo(0, OptionType::kUInt64T)
.Parse(config_options, "b", "x", &u64));
ASSERT_NOK(OptionTypeInfo(0, OptionType::kSizeT)
.Parse(config_options, "b", "x", &sz));
ASSERT_NOK(OptionTypeInfo(0, OptionType::kDouble) ASSERT_NOK(OptionTypeInfo(0, OptionType::kDouble)
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&d))); .Parse(config_options, "b", "x", &d));
// Don't know how to convert Unknowns to anything else // Don't know how to convert Unknowns to anything else
ASSERT_NOK(OptionTypeInfo(0, OptionType::kUnknown) ASSERT_NOK(OptionTypeInfo(0, OptionType::kUnknown)
.Parse(config_options, "b", "x", reinterpret_cast<char*>(&d))); .Parse(config_options, "b", "x", &d));
// Verify that if the parse function throws an exception, it is also trapped // Verify that if the parse function throws an exception, it is also trapped
OptionTypeInfo func_info(0, OptionType::kUnknown, OptionTypeInfo func_info(0, OptionType::kUnknown,
OptionVerificationType::kNormal, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0, OptionTypeFlags::kNone,
[](const ConfigOptions&, const std::string&, [](const ConfigOptions&, const std::string&,
const std::string& value, char* addr) { const std::string& value, char* addr) {
auto ptr = reinterpret_cast<int*>(addr); auto ptr = reinterpret_cast<int*>(addr);
*ptr = ParseInt(value); *ptr = ParseInt(value);
return Status::OK(); return Status::OK();
}); });
ASSERT_OK( ASSERT_OK(func_info.Parse(config_options, "b", "1", &i));
func_info.Parse(config_options, "b", "1", reinterpret_cast<char*>(&i))); ASSERT_NOK(func_info.Parse(config_options, "b", "x", &i));
ASSERT_NOK(
func_info.Parse(config_options, "b", "x", reinterpret_cast<char*>(&i)));
} }
TEST_F(OptionTypeInfoTest, TestParseFunc) { TEST_F(OptionTypeInfoTest, TestParseFunc) {
OptionTypeInfo opt_info( OptionTypeInfo opt_info(
0, OptionType::kUnknown, OptionVerificationType::kNormal, 0, OptionType::kUnknown, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0, OptionTypeFlags::kNone,
[](const ConfigOptions& /*opts*/, const std::string& name, [](const ConfigOptions& /*opts*/, const std::string& name,
const std::string& value, char* addr) { const std::string& value, char* addr) {
auto ptr = reinterpret_cast<std::string*>(addr); auto ptr = reinterpret_cast<std::string*>(addr);
@ -3332,17 +3422,15 @@ TEST_F(OptionTypeInfoTest, TestParseFunc) {
}); });
ConfigOptions config_options; ConfigOptions config_options;
std::string base; std::string base;
ASSERT_OK(opt_info.Parse(config_options, "World", "Hello", ASSERT_OK(opt_info.Parse(config_options, "World", "Hello", &base));
reinterpret_cast<char*>(&base)));
ASSERT_EQ(base, "Hello World"); ASSERT_EQ(base, "Hello World");
ASSERT_NOK(opt_info.Parse(config_options, "Oops", "Hello", ASSERT_NOK(opt_info.Parse(config_options, "Oops", "Hello", &base));
reinterpret_cast<char*>(&base)));
} }
TEST_F(OptionTypeInfoTest, TestSerializeFunc) { TEST_F(OptionTypeInfoTest, TestSerializeFunc) {
OptionTypeInfo opt_info( OptionTypeInfo opt_info(
0, OptionType::kString, OptionVerificationType::kNormal, 0, OptionType::kString, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0, nullptr, OptionTypeFlags::kNone, nullptr,
[](const ConfigOptions& /*opts*/, const std::string& name, [](const ConfigOptions& /*opts*/, const std::string& name,
const char* /*addr*/, std::string* value) { const char* /*addr*/, std::string* value) {
if (name == "Oops") { if (name == "Oops") {
@ -3356,17 +3444,15 @@ TEST_F(OptionTypeInfoTest, TestSerializeFunc) {
ConfigOptions config_options; ConfigOptions config_options;
std::string base; std::string base;
std::string value; std::string value;
ASSERT_OK(opt_info.Serialize(config_options, "Hello", ASSERT_OK(opt_info.Serialize(config_options, "Hello", &base, &value));
reinterpret_cast<char*>(&base), &value));
ASSERT_EQ(value, "Hello"); ASSERT_EQ(value, "Hello");
ASSERT_NOK(opt_info.Serialize(config_options, "Oops", ASSERT_NOK(opt_info.Serialize(config_options, "Oops", &base, &value));
reinterpret_cast<char*>(&base), &value));
} }
TEST_F(OptionTypeInfoTest, TestEqualsFunc) { TEST_F(OptionTypeInfoTest, TestEqualsFunc) {
OptionTypeInfo opt_info( OptionTypeInfo opt_info(
0, OptionType::kInt, OptionVerificationType::kNormal, 0, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0, nullptr, nullptr, OptionTypeFlags::kNone, nullptr, nullptr,
[](const ConfigOptions& /*opts*/, const std::string& name, [](const ConfigOptions& /*opts*/, const std::string& name,
const char* addr1, const char* addr2, std::string* mismatch) { const char* addr1, const char* addr2, std::string* mismatch) {
auto i1 = *(reinterpret_cast<const int*>(addr1)); auto i1 = *(reinterpret_cast<const int*>(addr1));
@ -3387,69 +3473,56 @@ TEST_F(OptionTypeInfoTest, TestEqualsFunc) {
int int1 = 100; int int1 = 100;
int int2 = 200; int int2 = 200;
std::string mismatch; std::string mismatch;
ASSERT_TRUE(opt_info.AreEqual( ASSERT_TRUE(opt_info.AreEqual(config_options, "LT", &int1, &int2, &mismatch));
config_options, "LT", reinterpret_cast<const char*>(&int1),
reinterpret_cast<const char*>(&int2), &mismatch));
ASSERT_EQ(mismatch, ""); ASSERT_EQ(mismatch, "");
ASSERT_FALSE(opt_info.AreEqual(config_options, "GT", ASSERT_FALSE(
reinterpret_cast<char*>(&int1), opt_info.AreEqual(config_options, "GT", &int1, &int2, &mismatch));
reinterpret_cast<char*>(&int2), &mismatch));
ASSERT_EQ(mismatch, "GT"); ASSERT_EQ(mismatch, "GT");
ASSERT_FALSE(opt_info.AreEqual(config_options, "NO", ASSERT_FALSE(
reinterpret_cast<char*>(&int1), opt_info.AreEqual(config_options, "NO", &int1, &int2, &mismatch));
reinterpret_cast<char*>(&int2), &mismatch));
ASSERT_EQ(mismatch, "NO???"); ASSERT_EQ(mismatch, "NO???");
} }
TEST_F(OptionTypeInfoTest, TestOptionFlags) { TEST_F(OptionTypeInfoTest, TestOptionFlags) {
OptionTypeInfo opt_none(0, OptionType::kString, OptionTypeInfo opt_none(0, OptionType::kString,
OptionVerificationType::kNormal, OptionVerificationType::kNormal,
OptionTypeFlags::kDontSerialize, 0); OptionTypeFlags::kDontSerialize);
OptionTypeInfo opt_never(0, OptionType::kString, OptionTypeInfo opt_never(0, OptionType::kString,
OptionVerificationType::kNormal, OptionVerificationType::kNormal,
OptionTypeFlags::kCompareNever, 0); OptionTypeFlags::kCompareNever);
OptionTypeInfo opt_alias(0, OptionType::kString, OptionTypeInfo opt_alias(0, OptionType::kString,
OptionVerificationType::kAlias, OptionVerificationType::kAlias,
OptionTypeFlags::kNone, 0); OptionTypeFlags::kNone);
OptionTypeInfo opt_deprecated(0, OptionType::kString, OptionTypeInfo opt_deprecated(0, OptionType::kString,
OptionVerificationType::kDeprecated, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0); OptionTypeFlags::kNone);
ConfigOptions config_options; ConfigOptions config_options;
std::string opts_str;
std::string base = "base"; std::string base = "base";
std::string comp = "comp"; std::string comp = "comp";
// If marked string none, the serialization returns okay but does nothing // If marked string none, the serialization returns not supported
ASSERT_OK(opt_none.Serialize(config_options, "None", ASSERT_NOK(opt_none.Serialize(config_options, "None", &base, &opts_str));
reinterpret_cast<char*>(&base), &base));
// If marked never compare, they match even when they do not // If marked never compare, they match even when they do not
ASSERT_TRUE(opt_never.AreEqual(config_options, "Never", ASSERT_TRUE(opt_never.AreEqual(config_options, "Never", &base, &comp, &base));
reinterpret_cast<char*>(&base), ASSERT_FALSE(opt_none.AreEqual(config_options, "Never", &base, &comp, &base));
reinterpret_cast<char*>(&comp), &base));
ASSERT_FALSE(opt_none.AreEqual(config_options, "Never",
reinterpret_cast<char*>(&base),
reinterpret_cast<char*>(&comp), &base));
// An alias can change the value via parse, but does nothing on serialize on // An alias can change the value via parse, but does nothing on serialize on
// match // match
std::string result; std::string result;
ASSERT_OK(opt_alias.Parse(config_options, "Alias", "Alias", ASSERT_OK(opt_alias.Parse(config_options, "Alias", "Alias",
reinterpret_cast<char*>(&base))); reinterpret_cast<char*>(&base)));
ASSERT_OK(opt_alias.Serialize(config_options, "Alias", ASSERT_OK(opt_alias.Serialize(config_options, "Alias", &base, &result));
reinterpret_cast<char*>(&base), &result)); ASSERT_TRUE(
ASSERT_TRUE(opt_alias.AreEqual(config_options, "Alias", opt_alias.AreEqual(config_options, "Alias", &base, &comp, &result));
reinterpret_cast<char*>(&base),
reinterpret_cast<char*>(&comp), &result));
ASSERT_EQ(base, "Alias"); ASSERT_EQ(base, "Alias");
ASSERT_NE(base, comp); ASSERT_NE(base, comp);
// Deprecated options do nothing on any of the commands // Deprecated options do nothing on any of the commands
ASSERT_OK(opt_deprecated.Parse(config_options, "Alias", "Deprecated", ASSERT_OK(opt_deprecated.Parse(config_options, "Alias", "Deprecated", &base));
reinterpret_cast<char*>(&base))); ASSERT_OK(opt_deprecated.Serialize(config_options, "Alias", &base, &result));
ASSERT_OK(opt_deprecated.Serialize(config_options, "Alias", ASSERT_TRUE(
reinterpret_cast<char*>(&base), &result)); opt_deprecated.AreEqual(config_options, "Alias", &base, &comp, &result));
ASSERT_TRUE(opt_deprecated.AreEqual(config_options, "Alias",
reinterpret_cast<char*>(&base),
reinterpret_cast<char*>(&comp), &result));
ASSERT_EQ(base, "Alias"); ASSERT_EQ(base, "Alias");
ASSERT_NE(base, comp); ASSERT_NE(base, comp);
} }
@ -3468,25 +3541,18 @@ TEST_F(OptionTypeInfoTest, TestCustomEnum) {
e2 = TestEnum::kA; e2 = TestEnum::kA;
ASSERT_OK( ASSERT_OK(opt_info.Parse(config_options, "", "B", &e1));
opt_info.Parse(config_options, "", "B", reinterpret_cast<char*>(&e1))); ASSERT_OK(opt_info.Serialize(config_options, "", &e1, &result));
ASSERT_OK(opt_info.Serialize(config_options, "", reinterpret_cast<char*>(&e1),
&result));
ASSERT_EQ(e1, TestEnum::kB); ASSERT_EQ(e1, TestEnum::kB);
ASSERT_EQ(result, "B"); ASSERT_EQ(result, "B");
ASSERT_FALSE(opt_info.AreEqual(config_options, "Enum", ASSERT_FALSE(opt_info.AreEqual(config_options, "Enum", &e1, &e2, &mismatch));
reinterpret_cast<char*>(&e1),
reinterpret_cast<char*>(&e2), &mismatch));
ASSERT_EQ(mismatch, "Enum"); ASSERT_EQ(mismatch, "Enum");
TestAndCompareOption(config_options, opt_info, "", "C", TestAndCompareOption(config_options, opt_info, "", "C", &e1, &e2);
reinterpret_cast<char*>(&e1),
reinterpret_cast<char*>(&e2));
ASSERT_EQ(e2, TestEnum::kC); ASSERT_EQ(e2, TestEnum::kC);
ASSERT_NOK( ASSERT_NOK(opt_info.Parse(config_options, "", "D", &e1));
opt_info.Parse(config_options, "", "D", reinterpret_cast<char*>(&e1)));
ASSERT_EQ(e1, TestEnum::kC); ASSERT_EQ(e1, TestEnum::kC);
} }
@ -3553,21 +3619,20 @@ TEST_F(OptionTypeInfoTest, TestStruct) {
}; };
OptionTypeInfo basic_info = OptionTypeInfo::Struct( OptionTypeInfo basic_info = OptionTypeInfo::Struct(
"b", &basic_type_map, 0, OptionVerificationType::kNormal, "b", &basic_type_map, 0, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable, 0); OptionTypeFlags::kMutable);
std::unordered_map<std::string, OptionTypeInfo> extended_type_map = { std::unordered_map<std::string, OptionTypeInfo> extended_type_map = {
{"j", {offsetof(struct Extended, j), OptionType::kInt}}, {"j", {offsetof(struct Extended, j), OptionType::kInt}},
{"b", OptionTypeInfo::Struct( {"b", OptionTypeInfo::Struct(
"b", &basic_type_map, offsetof(struct Extended, b), "b", &basic_type_map, offsetof(struct Extended, b),
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0)}, OptionVerificationType::kNormal, OptionTypeFlags::kNone)},
{"m", OptionTypeInfo::Struct( {"m", OptionTypeInfo::Struct(
"m", &basic_type_map, offsetof(struct Extended, b), "m", &basic_type_map, offsetof(struct Extended, b),
OptionVerificationType::kNormal, OptionTypeFlags::kMutable, OptionVerificationType::kNormal, OptionTypeFlags::kMutable)},
offsetof(struct Extended, b))},
}; };
OptionTypeInfo extended_info = OptionTypeInfo::Struct( OptionTypeInfo extended_info = OptionTypeInfo::Struct(
"e", &extended_type_map, 0, OptionVerificationType::kNormal, "e", &extended_type_map, 0, OptionVerificationType::kNormal,
OptionTypeFlags::kMutable, 0); OptionTypeFlags::kMutable);
Extended e1, e2; Extended e1, e2;
ConfigOptions config_options; ConfigOptions config_options;
std::string mismatch; std::string mismatch;
@ -3583,30 +3648,24 @@ TEST_F(OptionTypeInfoTest, TestStruct) {
ASSERT_EQ(e1.b.i, 55); ASSERT_EQ(e1.b.i, 55);
e1.b.i = 0; e1.b.i = 0;
auto e1bc = reinterpret_cast<char*>(&e1.b);
auto e2bc = reinterpret_cast<char*>(&e2.b);
ASSERT_FALSE(basic_info.AreEqual(config_options, "b", e1bc, e2bc, &mismatch)); ASSERT_FALSE(
basic_info.AreEqual(config_options, "b", &e1.b, &e2.b, &mismatch));
ASSERT_EQ(mismatch, "b.i"); ASSERT_EQ(mismatch, "b.i");
mismatch.clear(); mismatch.clear();
ASSERT_FALSE( ASSERT_FALSE(
basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); basic_info.AreEqual(config_options, "b.i", &e1.b, &e2.b, &mismatch));
ASSERT_EQ(mismatch, "b.i"); ASSERT_EQ(mismatch, "b.i");
mismatch.clear(); mismatch.clear();
ASSERT_FALSE(basic_info.AreEqual(config_options, "i", e1bc, e2bc, &mismatch)); ASSERT_FALSE(
basic_info.AreEqual(config_options, "i", &e1.b, &e2.b, &mismatch));
ASSERT_EQ(mismatch, "b.i"); ASSERT_EQ(mismatch, "b.i");
mismatch.clear(); mismatch.clear();
e1 = e2; e1 = e2;
ASSERT_NOK(basic_info.Parse(config_options, "b", "{i=33;s=33;j=44}", e1bc)); ASSERT_NOK(basic_info.Parse(config_options, "b", "{i=33;s=33;j=44}", &e1.b));
ASSERT_TRUE( ASSERT_NOK(basic_info.Parse(config_options, "b.j", "44", &e1.b));
basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch)); ASSERT_NOK(basic_info.Parse(config_options, "j", "44", &e1.b));
ASSERT_NOK(basic_info.Parse(config_options, "b.j", "44", e1bc));
ASSERT_TRUE(
basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch));
ASSERT_NOK(basic_info.Parse(config_options, "j", "44", e1bc));
ASSERT_TRUE(
basic_info.AreEqual(config_options, "b.i", e1bc, e2bc, &mismatch));
TestAndCompareOption(config_options, extended_info, "e", TestAndCompareOption(config_options, extended_info, "e",
"b={i=55;s=55}; j=22;", &e1, &e2); "b={i=55;s=55}; j=22;", &e1, &e2);
@ -3626,7 +3685,7 @@ TEST_F(OptionTypeInfoTest, TestStruct) {
TEST_F(OptionTypeInfoTest, TestVectorType) { TEST_F(OptionTypeInfoTest, TestVectorType) {
OptionTypeInfo vec_info = OptionTypeInfo::Vector<std::string>( OptionTypeInfo vec_info = OptionTypeInfo::Vector<std::string>(
0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0, 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone,
{0, OptionType::kString}); {0, OptionType::kString});
std::vector<std::string> vec1, vec2; std::vector<std::string> vec1, vec2;
std::string mismatch; std::string mismatch;
@ -3639,9 +3698,7 @@ TEST_F(OptionTypeInfoTest, TestVectorType) {
ASSERT_EQ(vec1[2], "c"); ASSERT_EQ(vec1[2], "c");
ASSERT_EQ(vec1[3], "d"); ASSERT_EQ(vec1[3], "d");
vec1[3] = "e"; vec1[3] = "e";
ASSERT_FALSE(vec_info.AreEqual(config_options, "v", ASSERT_FALSE(vec_info.AreEqual(config_options, "v", &vec1, &vec2, &mismatch));
reinterpret_cast<char*>(&vec1),
reinterpret_cast<char*>(&vec2), &mismatch));
ASSERT_EQ(mismatch, "v"); ASSERT_EQ(mismatch, "v");
// Test vectors with inner brackets // Test vectors with inner brackets
@ -3654,7 +3711,7 @@ TEST_F(OptionTypeInfoTest, TestVectorType) {
ASSERT_EQ(vec1[3], "d"); ASSERT_EQ(vec1[3], "d");
OptionTypeInfo bar_info = OptionTypeInfo::Vector<std::string>( OptionTypeInfo bar_info = OptionTypeInfo::Vector<std::string>(
0, OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0, 0, OptionVerificationType::kNormal, OptionTypeFlags::kNone,
{0, OptionType::kString}, '|'); {0, OptionType::kString}, '|');
TestAndCompareOption(config_options, vec_info, "v", "x|y|z", &vec1, &vec2); TestAndCompareOption(config_options, vec_info, "v", "x|y|z", &vec1, &vec2);
// Test vectors with inner vector // Test vectors with inner vector

@ -123,6 +123,7 @@ LIB_SOURCES = \
monitoring/thread_status_util.cc \ monitoring/thread_status_util.cc \
monitoring/thread_status_util_debug.cc \ monitoring/thread_status_util_debug.cc \
options/cf_options.cc \ options/cf_options.cc \
options/configurable.cc \
options/db_options.cc \ options/db_options.cc \
options/options.cc \ options/options.cc \
options/options_helper.cc \ options/options_helper.cc \
@ -174,6 +175,7 @@ LIB_SOURCES = \
table/sst_file_dumper.cc \ table/sst_file_dumper.cc \
table/sst_file_reader.cc \ table/sst_file_reader.cc \
table/sst_file_writer.cc \ table/sst_file_writer.cc \
table/table_factory.cc \
table/table_properties.cc \ table/table_properties.cc \
table/two_level_iterator.cc \ table/two_level_iterator.cc \
test_util/sync_point.cc \ test_util/sync_point.cc \
@ -439,6 +441,7 @@ TEST_MAIN_SOURCES = \
monitoring/statistics_test.cc \ monitoring/statistics_test.cc \
monitoring/stats_dump_scheduler_test.cc \ monitoring/stats_dump_scheduler_test.cc \
monitoring/stats_history_test.cc \ monitoring/stats_history_test.cc \
options/configurable_test.cc \
options/options_settable_test.cc \ options/options_settable_test.cc \
options/options_test.cc \ options/options_test.cc \
table/block_based/block_based_filter_block_test.cc \ table/block_based/block_based_filter_block_test.cc \

@ -77,7 +77,7 @@ TableBuilder* AdaptiveTableFactory::NewTableBuilder(
column_family_id, file); column_family_id, file);
} }
std::string AdaptiveTableFactory::GetPrintableTableOptions() const { std::string AdaptiveTableFactory::GetPrintableOptions() const {
std::string ret; std::string ret;
ret.reserve(20000); ret.reserve(20000);
const int kBufferSize = 200; const int kBufferSize = 200;
@ -87,13 +87,13 @@ std::string AdaptiveTableFactory::GetPrintableTableOptions() const {
snprintf(buffer, kBufferSize, " write factory (%s) options:\n%s\n", snprintf(buffer, kBufferSize, " write factory (%s) options:\n%s\n",
(table_factory_to_write_->Name() ? table_factory_to_write_->Name() (table_factory_to_write_->Name() ? table_factory_to_write_->Name()
: ""), : ""),
table_factory_to_write_->GetPrintableTableOptions().c_str()); table_factory_to_write_->GetPrintableOptions().c_str());
ret.append(buffer); ret.append(buffer);
} }
if (plain_table_factory_) { if (plain_table_factory_) {
snprintf(buffer, kBufferSize, " %s options:\n%s\n", snprintf(buffer, kBufferSize, " %s options:\n%s\n",
plain_table_factory_->Name() ? plain_table_factory_->Name() : "", plain_table_factory_->Name() ? plain_table_factory_->Name() : "",
plain_table_factory_->GetPrintableTableOptions().c_str()); plain_table_factory_->GetPrintableOptions().c_str());
ret.append(buffer); ret.append(buffer);
} }
if (block_based_table_factory_) { if (block_based_table_factory_) {
@ -101,13 +101,13 @@ std::string AdaptiveTableFactory::GetPrintableTableOptions() const {
buffer, kBufferSize, " %s options:\n%s\n", buffer, kBufferSize, " %s options:\n%s\n",
(block_based_table_factory_->Name() ? block_based_table_factory_->Name() (block_based_table_factory_->Name() ? block_based_table_factory_->Name()
: ""), : ""),
block_based_table_factory_->GetPrintableTableOptions().c_str()); block_based_table_factory_->GetPrintableOptions().c_str());
ret.append(buffer); ret.append(buffer);
} }
if (cuckoo_table_factory_) { if (cuckoo_table_factory_) {
snprintf(buffer, kBufferSize, " %s options:\n%s\n", snprintf(buffer, kBufferSize, " %s options:\n%s\n",
cuckoo_table_factory_->Name() ? cuckoo_table_factory_->Name() : "", cuckoo_table_factory_->Name() ? cuckoo_table_factory_->Name() : "",
cuckoo_table_factory_->GetPrintableTableOptions().c_str()); cuckoo_table_factory_->GetPrintableOptions().c_str());
ret.append(buffer); ret.append(buffer);
} }
return ret; return ret;

@ -44,14 +44,7 @@ class AdaptiveTableFactory : public TableFactory {
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,
uint32_t column_family_id, WritableFileWriter* file) const override; uint32_t column_family_id, WritableFileWriter* file) const override;
// Sanitizes the specified DB Options. std::string GetPrintableOptions() const override;
Status SanitizeOptions(
const DBOptions& /*db_opts*/,
const ColumnFamilyOptions& /*cf_opts*/) const override {
return Status::OK();
}
std::string GetPrintableTableOptions() const override;
private: private:
std::shared_ptr<TableFactory> table_factory_to_write_; std::shared_ptr<TableFactory> table_factory_to_write_;

@ -15,12 +15,12 @@
#include <memory> #include <memory>
#include <string> #include <string>
#include "options/options_helper.h" #include "options/configurable_helper.h"
#include "options/options_parser.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/cache.h" #include "rocksdb/cache.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/flush_block_policy.h" #include "rocksdb/flush_block_policy.h"
#include "rocksdb/utilities/options_type.h"
#include "table/block_based/block_based_table_builder.h" #include "table/block_based/block_based_table_builder.h"
#include "table/block_based/block_based_table_reader.h" #include "table/block_based/block_based_table_reader.h"
#include "table/format.h" #include "table/format.h"
@ -187,9 +187,11 @@ static std::unordered_map<std::string,
{"kShortenSeparatorsAndSuccessor", {"kShortenSeparatorsAndSuccessor",
BlockBasedTableOptions::IndexShorteningMode:: BlockBasedTableOptions::IndexShorteningMode::
kShortenSeparatorsAndSuccessor}}; kShortenSeparatorsAndSuccessor}};
#endif // ROCKSDB_LITE
static std::unordered_map<std::string, OptionTypeInfo> static std::unordered_map<std::string, OptionTypeInfo>
block_based_table_type_info = { block_based_table_type_info = {
#ifndef ROCKSDB_LITE
/* currently not supported /* currently not supported
std::shared_ptr<Cache> block_cache = nullptr; std::shared_ptr<Cache> block_cache = nullptr;
std::shared_ptr<Cache> block_cache_compressed = nullptr; std::shared_ptr<Cache> block_cache_compressed = nullptr;
@ -197,29 +199,29 @@ static std::unordered_map<std::string, OptionTypeInfo>
{"flush_block_policy_factory", {"flush_block_policy_factory",
{offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory),
OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName, OptionType::kFlushBlockPolicyFactory, OptionVerificationType::kByName,
OptionTypeFlags::kCompareNever, 0}}, OptionTypeFlags::kCompareNever}},
{"cache_index_and_filter_blocks", {"cache_index_and_filter_blocks",
{offsetof(struct BlockBasedTableOptions, {offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks), cache_index_and_filter_blocks),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"cache_index_and_filter_blocks_with_high_priority", {"cache_index_and_filter_blocks_with_high_priority",
{offsetof(struct BlockBasedTableOptions, {offsetof(struct BlockBasedTableOptions,
cache_index_and_filter_blocks_with_high_priority), cache_index_and_filter_blocks_with_high_priority),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"pin_l0_filter_and_index_blocks_in_cache", {"pin_l0_filter_and_index_blocks_in_cache",
{offsetof(struct BlockBasedTableOptions, {offsetof(struct BlockBasedTableOptions,
pin_l0_filter_and_index_blocks_in_cache), pin_l0_filter_and_index_blocks_in_cache),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"index_type", OptionTypeInfo::Enum<BlockBasedTableOptions::IndexType>( {"index_type", OptionTypeInfo::Enum<BlockBasedTableOptions::IndexType>(
offsetof(struct BlockBasedTableOptions, index_type), offsetof(struct BlockBasedTableOptions, index_type),
&block_base_table_index_type_string_map)}, &block_base_table_index_type_string_map)},
{"hash_index_allow_collision", {"hash_index_allow_collision",
{offsetof(struct BlockBasedTableOptions, hash_index_allow_collision), {offsetof(struct BlockBasedTableOptions, hash_index_allow_collision),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"data_block_index_type", {"data_block_index_type",
OptionTypeInfo::Enum<BlockBasedTableOptions::DataBlockIndexType>( OptionTypeInfo::Enum<BlockBasedTableOptions::DataBlockIndexType>(
offsetof(struct BlockBasedTableOptions, data_block_index_type), offsetof(struct BlockBasedTableOptions, data_block_index_type),
@ -232,50 +234,50 @@ static std::unordered_map<std::string, OptionTypeInfo>
{offsetof(struct BlockBasedTableOptions, {offsetof(struct BlockBasedTableOptions,
data_block_hash_table_util_ratio), data_block_hash_table_util_ratio),
OptionType::kDouble, OptionVerificationType::kNormal, OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"checksum", {"checksum",
{offsetof(struct BlockBasedTableOptions, checksum), {offsetof(struct BlockBasedTableOptions, checksum),
OptionType::kChecksumType, OptionVerificationType::kNormal, OptionType::kChecksumType, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"no_block_cache", {"no_block_cache",
{offsetof(struct BlockBasedTableOptions, no_block_cache), {offsetof(struct BlockBasedTableOptions, no_block_cache),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"block_size", {"block_size",
{offsetof(struct BlockBasedTableOptions, block_size), {offsetof(struct BlockBasedTableOptions, block_size),
OptionType::kSizeT, OptionVerificationType::kNormal, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kMutable}},
{"block_size_deviation", {"block_size_deviation",
{offsetof(struct BlockBasedTableOptions, block_size_deviation), {offsetof(struct BlockBasedTableOptions, block_size_deviation),
OptionType::kInt, OptionVerificationType::kNormal, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"block_restart_interval", {"block_restart_interval",
{offsetof(struct BlockBasedTableOptions, block_restart_interval), {offsetof(struct BlockBasedTableOptions, block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kMutable}},
{"index_block_restart_interval", {"index_block_restart_interval",
{offsetof(struct BlockBasedTableOptions, index_block_restart_interval), {offsetof(struct BlockBasedTableOptions, index_block_restart_interval),
OptionType::kInt, OptionVerificationType::kNormal, OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"index_per_partition", {"index_per_partition",
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated, {0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"metadata_block_size", {"metadata_block_size",
{offsetof(struct BlockBasedTableOptions, metadata_block_size), {offsetof(struct BlockBasedTableOptions, metadata_block_size),
OptionType::kUInt64T, OptionVerificationType::kNormal, OptionType::kUInt64T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"partition_filters", {"partition_filters",
{offsetof(struct BlockBasedTableOptions, partition_filters), {offsetof(struct BlockBasedTableOptions, partition_filters),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"optimize_filters_for_memory", {"optimize_filters_for_memory",
{offsetof(struct BlockBasedTableOptions, optimize_filters_for_memory), {offsetof(struct BlockBasedTableOptions, optimize_filters_for_memory),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"filter_policy", {"filter_policy",
{offsetof(struct BlockBasedTableOptions, filter_policy), {offsetof(struct BlockBasedTableOptions, filter_policy),
OptionType::kUnknown, OptionVerificationType::kByNameAllowFromNull, OptionType::kUnknown, OptionVerificationType::kByNameAllowFromNull,
OptionTypeFlags::kNone, 0, OptionTypeFlags::kNone,
// Parses the Filter policy // Parses the Filter policy
[](const ConfigOptions& opts, const std::string&, [](const ConfigOptions& opts, const std::string&,
const std::string& value, char* addr) { const std::string& value, char* addr) {
@ -317,39 +319,39 @@ static std::unordered_map<std::string, OptionTypeInfo>
{"whole_key_filtering", {"whole_key_filtering",
{offsetof(struct BlockBasedTableOptions, whole_key_filtering), {offsetof(struct BlockBasedTableOptions, whole_key_filtering),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"skip_table_builder_flush", {"skip_table_builder_flush",
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated, {0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"format_version", {"format_version",
{offsetof(struct BlockBasedTableOptions, format_version), {offsetof(struct BlockBasedTableOptions, format_version),
OptionType::kUInt32T, OptionVerificationType::kNormal, OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"verify_compression", {"verify_compression",
{offsetof(struct BlockBasedTableOptions, verify_compression), {offsetof(struct BlockBasedTableOptions, verify_compression),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"read_amp_bytes_per_bit", {"read_amp_bytes_per_bit",
{offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit), {offsetof(struct BlockBasedTableOptions, read_amp_bytes_per_bit),
OptionType::kSizeT, OptionVerificationType::kNormal, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"enable_index_compression", {"enable_index_compression",
{offsetof(struct BlockBasedTableOptions, enable_index_compression), {offsetof(struct BlockBasedTableOptions, enable_index_compression),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"block_align", {"block_align",
{offsetof(struct BlockBasedTableOptions, block_align), {offsetof(struct BlockBasedTableOptions, block_align),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"pin_top_level_index_and_filter", {"pin_top_level_index_and_filter",
{offsetof(struct BlockBasedTableOptions, {offsetof(struct BlockBasedTableOptions,
pin_top_level_index_and_filter), pin_top_level_index_and_filter),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"block_cache", {"block_cache",
{offsetof(struct BlockBasedTableOptions, block_cache), {offsetof(struct BlockBasedTableOptions, block_cache),
OptionType::kUnknown, OptionVerificationType::kNormal, OptionType::kUnknown, OptionVerificationType::kNormal,
(OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), 0, (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize),
// Parses the input vsalue as a Cache // Parses the input vsalue as a Cache
[](const ConfigOptions& opts, const std::string&, [](const ConfigOptions& opts, const std::string&,
const std::string& value, char* addr) { const std::string& value, char* addr) {
@ -359,21 +361,27 @@ static std::unordered_map<std::string, OptionTypeInfo>
{"block_cache_compressed", {"block_cache_compressed",
{offsetof(struct BlockBasedTableOptions, block_cache_compressed), {offsetof(struct BlockBasedTableOptions, block_cache_compressed),
OptionType::kUnknown, OptionVerificationType::kNormal, OptionType::kUnknown, OptionVerificationType::kNormal,
(OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize), 0, (OptionTypeFlags::kCompareNever | OptionTypeFlags::kDontSerialize),
// Parses the input vsalue as a Cache // Parses the input vsalue as a Cache
[](const ConfigOptions& opts, const std::string&, [](const ConfigOptions& opts, const std::string&,
const std::string& value, char* addr) { const std::string& value, char* addr) {
auto* cache = reinterpret_cast<std::shared_ptr<Cache>*>(addr); auto* cache = reinterpret_cast<std::shared_ptr<Cache>*>(addr);
return Cache::CreateFromString(opts, value, cache); return Cache::CreateFromString(opts, value, cache);
}}}, }}},
};
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
};
// TODO(myabandeh): We should return an error instead of silently changing the // TODO(myabandeh): We should return an error instead of silently changing the
// options // options
BlockBasedTableFactory::BlockBasedTableFactory( BlockBasedTableFactory::BlockBasedTableFactory(
const BlockBasedTableOptions& _table_options) const BlockBasedTableOptions& _table_options)
: table_options_(_table_options) { : table_options_(_table_options) {
InitializeOptions();
ConfigurableHelper::RegisterOptions(*this, &table_options_,
&block_based_table_type_info);
}
void BlockBasedTableFactory::InitializeOptions() {
if (table_options_.flush_block_policy_factory == nullptr) { if (table_options_.flush_block_policy_factory == nullptr) {
table_options_.flush_block_policy_factory.reset( table_options_.flush_block_policy_factory.reset(
new FlushBlockBySizePolicyFactory()); new FlushBlockBySizePolicyFactory());
@ -411,6 +419,11 @@ BlockBasedTableFactory::BlockBasedTableFactory(
} }
} }
Status BlockBasedTableFactory::PrepareOptions(const ConfigOptions& opts) {
InitializeOptions();
return TableFactory::PrepareOptions(opts);
}
Status BlockBasedTableFactory::NewTableReader( Status BlockBasedTableFactory::NewTableReader(
const ReadOptions& ro, const TableReaderOptions& table_reader_options, const ReadOptions& ro, const TableReaderOptions& table_reader_options,
std::unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size, std::unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
@ -449,7 +462,7 @@ TableBuilder* BlockBasedTableFactory::NewTableBuilder(
return table_builder; return table_builder;
} }
Status BlockBasedTableFactory::SanitizeOptions( Status BlockBasedTableFactory::ValidateOptions(
const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts) const { const DBOptions& db_opts, const ColumnFamilyOptions& cf_opts) const {
if (table_options_.index_type == BlockBasedTableOptions::kHashSearch && if (table_options_.index_type == BlockBasedTableOptions::kHashSearch &&
cf_opts.prefix_extractor == nullptr) { cf_opts.prefix_extractor == nullptr) {
@ -501,10 +514,10 @@ Status BlockBasedTableFactory::SanitizeOptions(
"max_successive_merges larger than 0 is currently inconsistent with " "max_successive_merges larger than 0 is currently inconsistent with "
"unordered_write"); "unordered_write");
} }
return Status::OK(); return TableFactory::ValidateOptions(db_opts, cf_opts);
} }
std::string BlockBasedTableFactory::GetPrintableTableOptions() const { std::string BlockBasedTableFactory::GetPrintableOptions() const {
std::string ret; std::string ret;
ret.reserve(20000); ret.reserve(20000);
const int kBufferSize = 200; const int kBufferSize = 200;
@ -630,53 +643,74 @@ std::string BlockBasedTableFactory::GetPrintableTableOptions() const {
return ret; return ret;
} }
#ifndef ROCKSDB_LITE const void* BlockBasedTableFactory::GetOptionsPtr(
Status BlockBasedTableFactory::GetOptionString( const std::string& name) const {
const ConfigOptions& config_options, std::string* opt_string) const { if (name == kBlockCacheOpts()) {
assert(opt_string); if (table_options_.no_block_cache) {
opt_string->clear(); return nullptr;
return GetStringFromStruct(config_options, &table_options_, } else {
block_based_table_type_info, opt_string); return table_options_.block_cache.get();
} }
#else } else {
Status BlockBasedTableFactory::GetOptionString( return TableFactory::GetOptionsPtr(name);
const ConfigOptions& /*opts*/, std::string* /*opt_string*/) const {
return Status::OK();
} }
#endif // !ROCKSDB_LITE
const BlockBasedTableOptions& BlockBasedTableFactory::table_options() const {
return table_options_;
} }
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
namespace { // Take a default BlockBasedTableOptions "table_options" in addition to a
std::string ParseBlockBasedTableOption(const ConfigOptions& config_options, // map "opts_map" of option name to option value to construct the new
const std::string& name, // BlockBasedTableOptions "new_table_options".
const std::string& org_value, //
BlockBasedTableOptions* new_options) { // Below are the instructions of how to config some non-primitive-typed
const std::string& value = config_options.input_strings_escaped // options in BlockBasedTableOptions:
? UnescapeOptionString(org_value) //
: org_value; // * filter_policy:
const auto iter = block_based_table_type_info.find(name); // We currently only support the following FilterPolicy in the convenience
if (iter == block_based_table_type_info.end()) { // functions:
if (config_options.ignore_unknown_options) { // - BloomFilter: use "bloomfilter:[bits_per_key]:[use_block_based_builder]"
return ""; // to specify BloomFilter. The above string is equivalent to calling
} else { // NewBloomFilterPolicy(bits_per_key, use_block_based_builder).
return "Unrecognized option"; // [Example]:
} // - Pass {"filter_policy", "bloomfilter:4:true"} in
// GetBlockBasedTableOptionsFromMap to use a BloomFilter with 4-bits
// per key and use_block_based_builder enabled.
//
// * block_cache / block_cache_compressed:
// We currently only support LRU cache in the GetOptions API. The LRU
// cache can be set by directly specifying its size.
// [Example]:
// - Passing {"block_cache", "1M"} in GetBlockBasedTableOptionsFromMap is
// equivalent to setting block_cache using NewLRUCache(1024 * 1024).
//
// @param table_options the default options of the output "new_table_options".
// @param opts_map an option name to value map for specifying how
// "new_table_options" should be set.
// @param new_table_options the resulting options based on "table_options"
// with the change specified in "opts_map".
// @param input_strings_escaped when set to true, each escaped characters
// prefixed by '\' in the values of the opts_map will be further converted
// back to the raw string before assigning to the associated options.
// @param ignore_unknown_options when set to true, unknown options are ignored
// instead of resulting in an unknown-option error.
// @return Status::OK() on success. Otherwise, a non-ok status indicating
// error will be returned, and "new_table_options" will be set to
// "table_options".
Status BlockBasedTableFactory::ParseOption(const ConfigOptions& config_options,
const OptionTypeInfo& opt_info,
const std::string& opt_name,
const std::string& opt_value,
void* opt_ptr) {
Status status = TableFactory::ParseOption(config_options, opt_info, opt_name,
opt_value, opt_ptr);
if (config_options.input_strings_escaped && !status.ok()) { // Got an error
// !input_strings_escaped indicates the old API, where everything is
// parsable.
if (opt_info.IsByName()) {
status = Status::OK();
} }
const auto& opt_info = iter->second;
Status s =
opt_info.Parse(config_options, iter->first, value,
reinterpret_cast<char*>(new_options) + opt_info.offset_);
if (s.ok()) {
return "";
} else {
return s.ToString();
} }
return status;
} }
} // namespace
Status GetBlockBasedTableOptionsFromString( Status GetBlockBasedTableOptionsFromString(
const BlockBasedTableOptions& table_options, const std::string& opts_str, const BlockBasedTableOptions& table_options, const std::string& opts_str,
@ -684,6 +718,7 @@ Status GetBlockBasedTableOptionsFromString(
ConfigOptions config_options; ConfigOptions config_options;
config_options.input_strings_escaped = false; config_options.input_strings_escaped = false;
config_options.ignore_unknown_options = false; config_options.ignore_unknown_options = false;
config_options.invoke_prepare_options = false;
return GetBlockBasedTableOptionsFromString(config_options, table_options, return GetBlockBasedTableOptionsFromString(config_options, table_options,
opts_str, new_table_options); opts_str, new_table_options);
} }
@ -708,6 +743,7 @@ Status GetBlockBasedTableOptionsFromMap(
ConfigOptions config_options; ConfigOptions config_options;
config_options.input_strings_escaped = input_strings_escaped; config_options.input_strings_escaped = input_strings_escaped;
config_options.ignore_unknown_options = ignore_unknown_options; config_options.ignore_unknown_options = ignore_unknown_options;
config_options.invoke_prepare_options = false;
return GetBlockBasedTableOptionsFromMap(config_options, table_options, return GetBlockBasedTableOptionsFromMap(config_options, table_options,
opts_map, new_table_options); opts_map, new_table_options);
@ -719,66 +755,14 @@ Status GetBlockBasedTableOptionsFromMap(
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
BlockBasedTableOptions* new_table_options) { BlockBasedTableOptions* new_table_options) {
assert(new_table_options); assert(new_table_options);
BlockBasedTableFactory bbtf(table_options);
Status s = bbtf.ConfigureFromMap(config_options, opts_map);
if (s.ok()) {
*new_table_options = *(bbtf.GetOptions<BlockBasedTableOptions>());
} else {
*new_table_options = table_options; *new_table_options = table_options;
for (const auto& o : opts_map) {
auto error_message = ParseBlockBasedTableOption(
config_options, o.first, o.second, new_table_options);
if (error_message != "") {
const auto iter = block_based_table_type_info.find(o.first);
if (iter == block_based_table_type_info.end() ||
!config_options
.input_strings_escaped || // !input_strings_escaped indicates
// the old API, where everything is
// parsable.
(!iter->second.IsByName() && !iter->second.IsDeprecated())) {
// Restore "new_options" to the default "base_options".
*new_table_options = table_options;
return Status::InvalidArgument("Can't parse BlockBasedTableOptions:",
o.first + " " + error_message);
}
}
} }
return Status::OK(); return s;
}
Status VerifyBlockBasedTableFactory(const ConfigOptions& config_options,
const BlockBasedTableFactory* base_tf,
const BlockBasedTableFactory* file_tf) {
if ((base_tf != nullptr) != (file_tf != nullptr) &&
config_options.sanity_level > ConfigOptions::kSanityLevelNone) {
return Status::Corruption(
"[RocksDBOptionsParser]: Inconsistent TableFactory class type");
}
if (base_tf == nullptr) {
return Status::OK();
}
assert(file_tf != nullptr);
const auto& base_opt = base_tf->table_options();
const auto& file_opt = file_tf->table_options();
std::string mismatch;
for (auto& pair : block_based_table_type_info) {
// We skip checking deprecated variables as they might
// contain random values since they might not be initialized
if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) {
const char* base_addr =
reinterpret_cast<const char*>(&base_opt) + pair.second.offset_;
const char* file_addr =
reinterpret_cast<const char*>(&file_opt) + pair.second.offset_;
if (!pair.second.AreEqual(config_options, pair.first, base_addr,
file_addr, &mismatch) &&
!pair.second.AreEqualByName(config_options, pair.first, base_addr,
file_addr)) {
return Status::Corruption(
"[RocksDBOptionsParser]: "
"failed the verification on BlockBasedTableOptions::",
pair.first);
}
}
}
return Status::OK();
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
@ -787,7 +771,6 @@ TableFactory* NewBlockBasedTableFactory(
return new BlockBasedTableFactory(_table_options); return new BlockBasedTableFactory(_table_options);
} }
const std::string BlockBasedTableFactory::kName = "BlockBasedTable";
const std::string BlockBasedTablePropertyNames::kIndexType = const std::string BlockBasedTablePropertyNames::kIndexType =
"rocksdb.block.based.table.index.type"; "rocksdb.block.based.table.index.type";
const std::string BlockBasedTablePropertyNames::kWholeKeyFiltering = const std::string BlockBasedTablePropertyNames::kWholeKeyFiltering =

@ -46,7 +46,7 @@ class BlockBasedTableFactory : public TableFactory {
~BlockBasedTableFactory() {} ~BlockBasedTableFactory() {}
const char* Name() const override { return kName.c_str(); } const char* Name() const override { return kBlockBasedTableName(); }
using TableFactory::NewTableReader; using TableFactory::NewTableReader;
Status NewTableReader( Status NewTableReader(
@ -59,24 +59,26 @@ class BlockBasedTableFactory : public TableFactory {
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,
uint32_t column_family_id, WritableFileWriter* file) const override; uint32_t column_family_id, WritableFileWriter* file) const override;
// Sanitizes the specified DB Options. // Valdates the specified DB Options.
Status SanitizeOptions(const DBOptions& db_opts, Status ValidateOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const override; const ColumnFamilyOptions& cf_opts) const override;
Status PrepareOptions(const ConfigOptions& opts) override;
std::string GetPrintableTableOptions() const override; std::string GetPrintableOptions() const override;
Status GetOptionString(const ConfigOptions& config_options,
std::string* opt_string) const override;
const BlockBasedTableOptions& table_options() const;
void* GetOptions() override { return &table_options_; }
bool IsDeleteRangeSupported() const override { return true; } bool IsDeleteRangeSupported() const override { return true; }
TailPrefetchStats* tail_prefetch_stats() { return &tail_prefetch_stats_; } TailPrefetchStats* tail_prefetch_stats() { return &tail_prefetch_stats_; }
static const std::string kName; protected:
const void* GetOptionsPtr(const std::string& name) const override;
#ifndef ROCKSDB_LITE
Status ParseOption(const ConfigOptions& config_options,
const OptionTypeInfo& opt_info,
const std::string& opt_name, const std::string& opt_value,
void* opt_ptr) override;
#endif
void InitializeOptions();
private: private:
BlockBasedTableOptions table_options_; BlockBasedTableOptions table_options_;
@ -87,10 +89,4 @@ extern const std::string kHashIndexPrefixesBlock;
extern const std::string kHashIndexPrefixesMetadataBlock; extern const std::string kHashIndexPrefixesMetadataBlock;
extern const std::string kPropTrue; extern const std::string kPropTrue;
extern const std::string kPropFalse; extern const std::string kPropFalse;
#ifndef ROCKSDB_LITE
extern Status VerifyBlockBasedTableFactory(
const ConfigOptions& config_options, const BlockBasedTableFactory* base_tf,
const BlockBasedTableFactory* file_tf);
#endif // !ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

@ -93,9 +93,12 @@ class BlockBasedTableReaderTest
std::unique_ptr<TableReader> table_reader; std::unique_ptr<TableReader> table_reader;
ReadOptions ro; ReadOptions ro;
ASSERT_OK(BlockBasedTable::Open(ro, ioptions, EnvOptions(), const auto* table_options =
table_factory_->table_options(), comparator, table_factory_->GetOptions<BlockBasedTableOptions>();
std::move(file), file_size, &table_reader)); ASSERT_NE(table_options, nullptr);
ASSERT_OK(BlockBasedTable::Open(ro, ioptions, EnvOptions(), *table_options,
comparator, std::move(file), file_size,
&table_reader));
table->reset(reinterpret_cast<BlockBasedTable*>(table_reader.release())); table->reset(reinterpret_cast<BlockBasedTable*>(table_reader.release()));
} }

@ -925,7 +925,7 @@ Status FilterPolicy::CreateFromString(
NewBloomFilterPolicy(bits_per_key, use_block_based_builder)); NewBloomFilterPolicy(bits_per_key, use_block_based_builder));
} }
} else { } else {
return Status::InvalidArgument("Invalid filter policy name ", value); return Status::NotFound("Invalid filter policy name ", value);
#else #else
} else { } else {
return Status::NotSupported("Cannot load filter policy in LITE mode ", return Status::NotSupported("Cannot load filter policy in LITE mode ",

@ -8,6 +8,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once #pragma once
#include "rocksdb/cache.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {

@ -276,9 +276,12 @@ class BlockFetcherTest : public testing::Test {
std::unique_ptr<TableReader> table_reader; std::unique_ptr<TableReader> table_reader;
ReadOptions ro; ReadOptions ro;
ASSERT_OK(BlockBasedTable::Open(ro, ioptions, EnvOptions(), const auto* table_options =
table_factory_.table_options(), comparator, table_factory_.GetOptions<BlockBasedTableOptions>();
std::move(file), file_size, &table_reader)); ASSERT_NE(table_options, nullptr);
ASSERT_OK(BlockBasedTable::Open(ro, ioptions, EnvOptions(), *table_options,
comparator, std::move(file), file_size,
&table_reader));
table->reset(reinterpret_cast<BlockBasedTable*>(table_reader.release())); table->reset(reinterpret_cast<BlockBasedTable*>(table_reader.release()));
} }

@ -7,6 +7,8 @@
#include "table/cuckoo/cuckoo_table_factory.h" #include "table/cuckoo/cuckoo_table_factory.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "options/configurable_helper.h"
#include "rocksdb/utilities/options_type.h"
#include "table/cuckoo/cuckoo_table_builder.h" #include "table/cuckoo/cuckoo_table_builder.h"
#include "table/cuckoo/cuckoo_table_reader.h" #include "table/cuckoo/cuckoo_table_reader.h"
@ -44,7 +46,7 @@ TableBuilder* CuckooTableFactory::NewTableBuilder(
table_builder_options.db_id, table_builder_options.db_session_id); table_builder_options.db_id, table_builder_options.db_session_id);
} }
std::string CuckooTableFactory::GetPrintableTableOptions() const { std::string CuckooTableFactory::GetPrintableOptions() const {
std::string ret; std::string ret;
ret.reserve(2000); ret.reserve(2000);
const int kBufferSize = 200; const int kBufferSize = 200;
@ -65,6 +67,38 @@ std::string CuckooTableFactory::GetPrintableTableOptions() const {
return ret; return ret;
} }
static std::unordered_map<std::string, OptionTypeInfo> cuckoo_table_type_info =
{
#ifndef ROCKSDB_LITE
{"hash_table_ratio",
{offsetof(struct CuckooTableOptions, hash_table_ratio),
OptionType::kDouble, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"max_search_depth",
{offsetof(struct CuckooTableOptions, max_search_depth),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"cuckoo_block_size",
{offsetof(struct CuckooTableOptions, cuckoo_block_size),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"identity_as_first_hash",
{offsetof(struct CuckooTableOptions, identity_as_first_hash),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"use_module_hash",
{offsetof(struct CuckooTableOptions, use_module_hash),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
#endif // ROCKSDB_LITE
};
CuckooTableFactory::CuckooTableFactory(const CuckooTableOptions& table_options)
: table_options_(table_options) {
ConfigurableHelper::RegisterOptions(*this, &table_options_,
&cuckoo_table_type_info);
}
TableFactory* NewCuckooTableFactory(const CuckooTableOptions& table_options) { TableFactory* NewCuckooTableFactory(const CuckooTableOptions& table_options) {
return new CuckooTableFactory(table_options); return new CuckooTableFactory(table_options);
} }

@ -52,11 +52,11 @@ static inline uint64_t CuckooHash(
// - Does not support prefix bloom filters. // - Does not support prefix bloom filters.
class CuckooTableFactory : public TableFactory { class CuckooTableFactory : public TableFactory {
public: public:
explicit CuckooTableFactory(const CuckooTableOptions& table_options) explicit CuckooTableFactory(
: table_options_(table_options) {} const CuckooTableOptions& table_option = CuckooTableOptions());
~CuckooTableFactory() {} ~CuckooTableFactory() {}
const char* Name() const override { return "CuckooTable"; } const char* Name() const override { return kCuckooTableName(); }
using TableFactory::NewTableReader; using TableFactory::NewTableReader;
Status NewTableReader( Status NewTableReader(
@ -69,21 +69,7 @@ class CuckooTableFactory : public TableFactory {
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,
uint32_t column_family_id, WritableFileWriter* file) const override; uint32_t column_family_id, WritableFileWriter* file) const override;
// Sanitizes the specified DB Options. std::string GetPrintableOptions() const override;
Status SanitizeOptions(
const DBOptions& /*db_opts*/,
const ColumnFamilyOptions& /*cf_opts*/) const override {
return Status::OK();
}
std::string GetPrintableTableOptions() const override;
void* GetOptions() override { return &table_options_; }
Status GetOptionString(const ConfigOptions& /*config_options*/,
std::string* /*opt_string*/) const override {
return Status::OK();
}
private: private:
CuckooTableOptions table_options_; CuckooTableOptions table_options_;

@ -62,13 +62,7 @@ class MockTableFactory : public TableFactory {
Status CreateMockTable(Env* env, const std::string& fname, Status CreateMockTable(Env* env, const std::string& fname,
stl_wrappers::KVMap file_contents); stl_wrappers::KVMap file_contents);
virtual Status SanitizeOptions( virtual std::string GetPrintableOptions() const override {
const DBOptions& /*db_opts*/,
const ColumnFamilyOptions& /*cf_opts*/) const override {
return Status::OK();
}
virtual std::string GetPrintableTableOptions() const override {
return std::string(); return std::string();
} }

@ -11,9 +11,10 @@
#include <memory> #include <memory>
#include "db/dbformat.h" #include "db/dbformat.h"
#include "options/options_helper.h" #include "options/configurable_helper.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/utilities/options_type.h"
#include "table/plain/plain_table_builder.h" #include "table/plain/plain_table_builder.h"
#include "table/plain/plain_table_reader.h" #include "table/plain/plain_table_reader.h"
#include "util/string_util.h" #include "util/string_util.h"
@ -22,31 +23,38 @@ namespace ROCKSDB_NAMESPACE {
static std::unordered_map<std::string, OptionTypeInfo> plain_table_type_info = { static std::unordered_map<std::string, OptionTypeInfo> plain_table_type_info = {
{"user_key_len", {"user_key_len",
{offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T, {offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"bloom_bits_per_key", {"bloom_bits_per_key",
{offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt, {offsetof(struct PlainTableOptions, bloom_bits_per_key), OptionType::kInt,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"hash_table_ratio", {"hash_table_ratio",
{offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble, {offsetof(struct PlainTableOptions, hash_table_ratio), OptionType::kDouble,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"index_sparseness", {"index_sparseness",
{offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT, {offsetof(struct PlainTableOptions, index_sparseness), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"huge_page_tlb_size", {"huge_page_tlb_size",
{offsetof(struct PlainTableOptions, huge_page_tlb_size), {offsetof(struct PlainTableOptions, huge_page_tlb_size),
OptionType::kSizeT, OptionVerificationType::kNormal, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"encoding_type", {"encoding_type",
{offsetof(struct PlainTableOptions, encoding_type), {offsetof(struct PlainTableOptions, encoding_type),
OptionType::kEncodingType, OptionVerificationType::kByName, OptionType::kEncodingType, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}, OptionTypeFlags::kNone}},
{"full_scan_mode", {"full_scan_mode",
{offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean, {offsetof(struct PlainTableOptions, full_scan_mode), OptionType::kBoolean,
OptionVerificationType::kNormal, OptionTypeFlags::kNone, 0}}, OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"store_index_in_file", {"store_index_in_file",
{offsetof(struct PlainTableOptions, store_index_in_file), {offsetof(struct PlainTableOptions, store_index_in_file),
OptionType::kBoolean, OptionVerificationType::kNormal, OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone, 0}}}; OptionTypeFlags::kNone}},
};
PlainTableFactory::PlainTableFactory(const PlainTableOptions& options)
: table_options_(options) {
ConfigurableHelper::RegisterOptions(*this, &table_options_,
&plain_table_type_info);
}
Status PlainTableFactory::NewTableReader( Status PlainTableFactory::NewTableReader(
const ReadOptions& /*ro*/, const TableReaderOptions& table_reader_options, const ReadOptions& /*ro*/, const TableReaderOptions& table_reader_options,
@ -80,7 +88,7 @@ TableBuilder* PlainTableFactory::NewTableBuilder(
table_builder_options.db_session_id); table_builder_options.db_session_id);
} }
std::string PlainTableFactory::GetPrintableTableOptions() const { std::string PlainTableFactory::GetPrintableOptions() const {
std::string ret; std::string ret;
ret.reserve(20000); ret.reserve(20000);
const int kBufferSize = 200; const int kBufferSize = 200;
@ -113,16 +121,13 @@ std::string PlainTableFactory::GetPrintableTableOptions() const {
return ret; return ret;
} }
const PlainTableOptions& PlainTableFactory::table_options() const {
return table_options_;
}
Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options, Status GetPlainTableOptionsFromString(const PlainTableOptions& table_options,
const std::string& opts_str, const std::string& opts_str,
PlainTableOptions* new_table_options) { PlainTableOptions* new_table_options) {
ConfigOptions config_options; ConfigOptions config_options;
config_options.input_strings_escaped = false; config_options.input_strings_escaped = false;
config_options.ignore_unknown_options = false; config_options.ignore_unknown_options = false;
config_options.invoke_prepare_options = false;
return GetPlainTableOptionsFromString(config_options, table_options, opts_str, return GetPlainTableOptionsFromString(config_options, table_options, opts_str,
new_table_options); new_table_options);
} }
@ -154,7 +159,7 @@ Status GetMemTableRepFactoryFromString(
MemTableRepFactory* mem_factory = nullptr; MemTableRepFactory* mem_factory = nullptr;
if (opts_list[0] == "skip_list") { if (opts_list[0] == "skip_list" || opts_list[0] == "SkipListFactory") {
// Expecting format // Expecting format
// skip_list:<lookahead> // skip_list:<lookahead>
if (2 == len) { if (2 == len) {
@ -163,7 +168,8 @@ Status GetMemTableRepFactoryFromString(
} else if (1 == len) { } else if (1 == len) {
mem_factory = new SkipListFactory(); mem_factory = new SkipListFactory();
} }
} else if (opts_list[0] == "prefix_hash") { } else if (opts_list[0] == "prefix_hash" ||
opts_list[0] == "HashSkipListRepFactory") {
// Expecting format // Expecting format
// prfix_hash:<hash_bucket_count> // prfix_hash:<hash_bucket_count>
if (2 == len) { if (2 == len) {
@ -172,7 +178,8 @@ Status GetMemTableRepFactoryFromString(
} else if (1 == len) { } else if (1 == len) {
mem_factory = NewHashSkipListRepFactory(); mem_factory = NewHashSkipListRepFactory();
} }
} else if (opts_list[0] == "hash_linkedlist") { } else if (opts_list[0] == "hash_linkedlist" ||
opts_list[0] == "HashLinkListRepFactory") {
// Expecting format // Expecting format
// hash_linkedlist:<hash_bucket_count> // hash_linkedlist:<hash_bucket_count>
if (2 == len) { if (2 == len) {
@ -181,7 +188,7 @@ Status GetMemTableRepFactoryFromString(
} else if (1 == len) { } else if (1 == len) {
mem_factory = NewHashLinkListRepFactory(); mem_factory = NewHashLinkListRepFactory();
} }
} else if (opts_list[0] == "vector") { } else if (opts_list[0] == "vector" || opts_list[0] == "VectorRepFactory") {
// Expecting format // Expecting format
// vector:<count> // vector:<count>
if (2 == len) { if (2 == len) {
@ -205,32 +212,6 @@ Status GetMemTableRepFactoryFromString(
return Status::OK(); return Status::OK();
} }
std::string ParsePlainTableOptions(const ConfigOptions& config_options,
const std::string& name,
const std::string& org_value,
PlainTableOptions* new_options) {
const std::string& value = config_options.input_strings_escaped
? UnescapeOptionString(org_value)
: org_value;
const auto iter = plain_table_type_info.find(name);
if (iter == plain_table_type_info.end()) {
if (config_options.ignore_unknown_options) {
return "";
} else {
return "Unrecognized option";
}
}
const auto& opt_info = iter->second;
Status s =
opt_info.Parse(config_options, name, value,
reinterpret_cast<char*>(new_options) + opt_info.offset_);
if (s.ok()) {
return "";
} else {
return s.ToString();
}
}
Status GetPlainTableOptionsFromMap( Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options, const PlainTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
@ -248,33 +229,21 @@ Status GetPlainTableOptionsFromMap(
const std::unordered_map<std::string, std::string>& opts_map, const std::unordered_map<std::string, std::string>& opts_map,
PlainTableOptions* new_table_options) { PlainTableOptions* new_table_options) {
assert(new_table_options); assert(new_table_options);
*new_table_options = table_options; PlainTableFactory ptf(table_options);
for (const auto& o : opts_map) { Status s = ptf.ConfigureFromMap(config_options, opts_map);
auto error_message = ParsePlainTableOptions(config_options, o.first, if (s.ok()) {
o.second, new_table_options); *new_table_options = *(ptf.GetOptions<PlainTableOptions>());
if (error_message != "") { } else {
const auto iter = plain_table_type_info.find(o.first);
if (iter == plain_table_type_info.end() ||
!config_options
.input_strings_escaped || // !input_strings_escaped indicates
// the old API, where everything is
// parsable.
(!iter->second.IsByName() && !iter->second.IsDeprecated())) {
// Restore "new_options" to the default "base_options". // Restore "new_options" to the default "base_options".
*new_table_options = table_options; *new_table_options = table_options;
return Status::InvalidArgument("Can't parse PlainTableOptions:",
o.first + " " + error_message);
} }
} return s;
}
return Status::OK();
} }
extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) { extern TableFactory* NewPlainTableFactory(const PlainTableOptions& options) {
return new PlainTableFactory(options); return new PlainTableFactory(options);
} }
const std::string PlainTableFactory::kName = "PlainTable";
const std::string PlainTablePropertyNames::kEncodingType = const std::string PlainTablePropertyNames::kEncodingType =
"rocksdb.plain.table.encoding.type"; "rocksdb.plain.table.encoding.type";

@ -10,8 +10,6 @@
#include <string> #include <string>
#include <stdint.h> #include <stdint.h>
#include "options/options_helper.h"
#include "rocksdb/options.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
@ -156,10 +154,9 @@ class PlainTableFactory : public TableFactory {
// page TLB and the page size if allocating from there. See comments of // page TLB and the page size if allocating from there. See comments of
// Arena::AllocateAligned() for details. // Arena::AllocateAligned() for details.
explicit PlainTableFactory( explicit PlainTableFactory(
const PlainTableOptions& _table_options = PlainTableOptions()) const PlainTableOptions& _table_options = PlainTableOptions());
: table_options_(_table_options) {}
const char* Name() const override { return kName.c_str(); } const char* Name() const override { return kPlainTableName(); }
using TableFactory::NewTableReader; using TableFactory::NewTableReader;
Status NewTableReader(const ReadOptions& ro, Status NewTableReader(const ReadOptions& ro,
const TableReaderOptions& table_reader_options, const TableReaderOptions& table_reader_options,
@ -171,28 +168,9 @@ class PlainTableFactory : public TableFactory {
const TableBuilderOptions& table_builder_options, const TableBuilderOptions& table_builder_options,
uint32_t column_family_id, WritableFileWriter* file) const override; uint32_t column_family_id, WritableFileWriter* file) const override;
std::string GetPrintableTableOptions() const override; std::string GetPrintableOptions() const override;
const PlainTableOptions& table_options() const;
static const char kValueTypeSeqId0 = char(~0); static const char kValueTypeSeqId0 = char(~0);
// Sanitizes the specified DB Options.
Status SanitizeOptions(
const DBOptions& /*db_opts*/,
const ColumnFamilyOptions& /*cf_opts*/) const override {
return Status::OK();
}
void* GetOptions() override { return &table_options_; }
Status GetOptionString(const ConfigOptions& /*config_options*/,
std::string* /*opt_string*/) const override {
return Status::OK();
}
static const std::string kName;
private: private:
PlainTableOptions table_options_; PlainTableOptions table_options_;
}; };

@ -156,7 +156,8 @@ Status SstFileDumper::NewTableReader(
// We need to turn off pre-fetching of index and filter nodes for // We need to turn off pre-fetching of index and filter nodes for
// BlockBasedTable // BlockBasedTable
if (BlockBasedTableFactory::kName == options_.table_factory->Name()) { if (options_.table_factory->IsInstanceOf(
TableFactory::kBlockBasedTableName())) {
return options_.table_factory->NewTableReader(t_opt, std::move(file_), return options_.table_factory->NewTableReader(t_opt, std::move(file_),
file_size, &table_reader_, file_size, &table_reader_,
/*enable_prefetch=*/false); /*enable_prefetch=*/false);

@ -0,0 +1,49 @@
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/convenience.h"
#include "rocksdb/table.h"
#include "table/block_based/block_based_table_factory.h"
#include "table/cuckoo/cuckoo_table_factory.h"
#include "table/plain/plain_table_factory.h"
namespace ROCKSDB_NAMESPACE {
Status TableFactory::CreateFromString(const ConfigOptions& config_options_in,
const std::string& id,
std::shared_ptr<TableFactory>* factory) {
Status status;
std::string name = id;
std::string existing_opts;
ConfigOptions config_options = config_options_in;
if (factory->get() != nullptr && name == factory->get()->Name()) {
config_options.delimiter = ";";
status = factory->get()->GetOptionString(config_options, &existing_opts);
if (!status.ok()) {
return status;
}
}
if (name == TableFactory::kBlockBasedTableName()) {
factory->reset(new BlockBasedTableFactory());
#ifndef ROCKSDB_LITE
} else if (name == TableFactory::kPlainTableName()) {
factory->reset(new PlainTableFactory());
} else if (name == TableFactory::kCuckooTableName()) {
factory->reset(new CuckooTableFactory());
#endif // ROCKSDB_LITE
} else {
return Status::NotSupported("Could not load table factory: ", name);
}
if (!existing_opts.empty()) {
config_options.invoke_prepare_options = false;
status = factory->get()->ConfigureFromString(config_options, existing_opts);
}
return status;
}
} // namespace ROCKSDB_NAMESPACE

@ -3008,7 +3008,7 @@ void ValidateBlockSizeDeviation(int value, int expected) {
BlockBasedTableFactory* factory = new BlockBasedTableFactory(table_options); BlockBasedTableFactory* factory = new BlockBasedTableFactory(table_options);
const BlockBasedTableOptions* normalized_table_options = const BlockBasedTableOptions* normalized_table_options =
(const BlockBasedTableOptions*)factory->GetOptions(); factory->GetOptions<BlockBasedTableOptions>();
ASSERT_EQ(normalized_table_options->block_size_deviation, expected); ASSERT_EQ(normalized_table_options->block_size_deviation, expected);
delete factory; delete factory;
@ -3020,7 +3020,7 @@ void ValidateBlockRestartInterval(int value, int expected) {
BlockBasedTableFactory* factory = new BlockBasedTableFactory(table_options); BlockBasedTableFactory* factory = new BlockBasedTableFactory(table_options);
const BlockBasedTableOptions* normalized_table_options = const BlockBasedTableOptions* normalized_table_options =
(const BlockBasedTableOptions*)factory->GetOptions(); factory->GetOptions<BlockBasedTableOptions>();
ASSERT_EQ(normalized_table_options->block_restart_interval, expected); ASSERT_EQ(normalized_table_options->block_restart_interval, expected);
delete factory; delete factory;

@ -23,6 +23,7 @@
#include "file/sequence_file_reader.h" #include "file/sequence_file_reader.h"
#include "file/writable_file_writer.h" #include "file/writable_file_writer.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/convenience.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "util/random.h" #include "util/random.h"

@ -4074,11 +4074,9 @@ class Benchmark {
options.compression_opts.parallel_threads = options.compression_opts.parallel_threads =
FLAGS_compression_parallel_threads; FLAGS_compression_parallel_threads;
// If this is a block based table, set some related options // If this is a block based table, set some related options
if (options.table_factory->Name() == BlockBasedTableFactory::kName && auto table_options =
options.table_factory->GetOptions() != nullptr) { options.table_factory->GetOptions<BlockBasedTableOptions>();
BlockBasedTableOptions* table_options = if (table_options != nullptr) {
reinterpret_cast<BlockBasedTableOptions*>(
options.table_factory->GetOptions());
if (FLAGS_cache_size) { if (FLAGS_cache_size) {
table_options->block_cache = cache_; table_options->block_cache = cache_;
} }

@ -40,7 +40,6 @@
#include "rocksdb/utilities/ldb_cmd.h" #include "rocksdb/utilities/ldb_cmd.h"
#include "rocksdb/write_batch.h" #include "rocksdb/write_batch.h"
#include "table/meta_blocks.h" #include "table/meta_blocks.h"
#include "table/plain/plain_table_factory.h"
#include "table/table_reader.h" #include "table/table_reader.h"
#include "tools/trace_analyzer_tool.h" #include "tools/trace_analyzer_tool.h"
#include "trace_replay/trace_replay.h" #include "trace_replay/trace_replay.h"

@ -188,8 +188,8 @@ TEST_F(TimerTest, AddAfterStartTest) {
} }
TEST_F(TimerTest, CancelRunningTask) { TEST_F(TimerTest, CancelRunningTask) {
static constexpr char kTestFuncName[] = "test_func";
const int kRepeatUs = 1 * kUsPerSec; const int kRepeatUs = 1 * kUsPerSec;
constexpr char kTestFuncName[] = "test_func";
Timer timer(mock_env_.get()); Timer timer(mock_env_.get());
ASSERT_TRUE(timer.Start()); ASSERT_TRUE(timer.Start());
int* value = new int; int* value = new int;

@ -38,12 +38,10 @@ class MemoryTest : public testing::Test {
void GetCachePointersFromTableFactory( void GetCachePointersFromTableFactory(
const TableFactory* factory, const TableFactory* factory,
std::unordered_set<const Cache*>* cache_set) { std::unordered_set<const Cache*>* cache_set) {
const BlockBasedTableFactory* bbtf = const auto bbto = factory->GetOptions<BlockBasedTableOptions>();
dynamic_cast<const BlockBasedTableFactory*>(factory); if (bbto != nullptr) {
if (bbtf != nullptr) { cache_set->insert(bbto->block_cache.get());
const auto bbt_opts = bbtf->table_options(); cache_set->insert(bbto->block_cache_compressed.get());
cache_set->insert(bbt_opts.block_cache.get());
cache_set->insert(bbt_opts.block_cache_compressed.get());
} }
} }

@ -47,11 +47,11 @@ Status LoadOptionsFromFile(const ConfigOptions& config_options,
cf_descs->push_back({cf_names[i], cf_opts[i]}); cf_descs->push_back({cf_names[i], cf_opts[i]});
if (cache != nullptr) { if (cache != nullptr) {
TableFactory* tf = cf_opts[i].table_factory.get(); TableFactory* tf = cf_opts[i].table_factory.get();
if (tf != nullptr && tf->GetOptions() != nullptr && if (tf != nullptr) {
tf->Name() == BlockBasedTableFactory::kName) { auto* opts = tf->GetOptions<BlockBasedTableOptions>();
auto* loaded_bbt_opt = if (opts != nullptr) {
reinterpret_cast<BlockBasedTableOptions*>(tf->GetOptions()); opts->block_cache = *cache;
loaded_bbt_opt->block_cache = *cache; }
} }
} }
} }

@ -43,10 +43,6 @@ class OptionsUtilTest : public testing::Test {
Random rnd_; Random rnd_;
}; };
bool IsBlockBasedTableFactory(TableFactory* tf) {
return tf->Name() == BlockBasedTableFactory::kName;
}
TEST_F(OptionsUtilTest, SaveAndLoad) { TEST_F(OptionsUtilTest, SaveAndLoad) {
const size_t kCFCount = 5; const size_t kCFCount = 5;
@ -80,11 +76,9 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name); ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name);
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
exact, cf_opts[i], loaded_cf_descs[i].options)); exact, cf_opts[i], loaded_cf_descs[i].options));
if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) {
ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory( ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory(
exact, cf_opts[i].table_factory.get(), exact, cf_opts[i].table_factory.get(),
loaded_cf_descs[i].options.table_factory.get())); loaded_cf_descs[i].options.table_factory.get()));
}
test::RandomInitCFOptions(&cf_opts[i], db_opt, &rnd_); test::RandomInitCFOptions(&cf_opts[i], db_opt, &rnd_);
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
exact, cf_opts[i], loaded_cf_descs[i].options)); exact, cf_opts[i], loaded_cf_descs[i].options));
@ -137,30 +131,28 @@ TEST_F(OptionsUtilTest, SaveAndLoadWithCacheCheck) {
ASSERT_OK(LoadOptionsFromFile(config_options, kFileName, &loaded_db_opt, ASSERT_OK(LoadOptionsFromFile(config_options, kFileName, &loaded_db_opt,
&loaded_cf_descs, &cache)); &loaded_cf_descs, &cache));
for (size_t i = 0; i < loaded_cf_descs.size(); i++) { for (size_t i = 0; i < loaded_cf_descs.size(); i++) {
if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { auto* loaded_bbt_opt =
auto* loaded_bbt_opt = reinterpret_cast<BlockBasedTableOptions*>( loaded_cf_descs[i]
loaded_cf_descs[i].options.table_factory->GetOptions()); .options.table_factory->GetOptions<BlockBasedTableOptions>();
// Expect the same cache will be loaded // Expect the same cache will be loaded
if (loaded_bbt_opt != nullptr) { if (loaded_bbt_opt != nullptr) {
ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get()); ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get());
} }
} }
}
// Test the old interface // Test the old interface
ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt, ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt,
&loaded_cf_descs, false, &cache)); &loaded_cf_descs, false, &cache));
for (size_t i = 0; i < loaded_cf_descs.size(); i++) { for (size_t i = 0; i < loaded_cf_descs.size(); i++) {
if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { auto* loaded_bbt_opt =
auto* loaded_bbt_opt = reinterpret_cast<BlockBasedTableOptions*>( loaded_cf_descs[i]
loaded_cf_descs[i].options.table_factory->GetOptions()); .options.table_factory->GetOptions<BlockBasedTableOptions>();
// Expect the same cache will be loaded // Expect the same cache will be loaded
if (loaded_bbt_opt != nullptr) { if (loaded_bbt_opt != nullptr) {
ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get()); ASSERT_EQ(loaded_bbt_opt->block_cache.get(), cache.get());
} }
} }
} }
}
namespace { namespace {
class DummyTableFactory : public TableFactory { class DummyTableFactory : public TableFactory {
@ -187,18 +179,13 @@ class DummyTableFactory : public TableFactory {
return nullptr; return nullptr;
} }
Status SanitizeOptions( Status ValidateOptions(
const DBOptions& /*db_opts*/, const DBOptions& /*db_opts*/,
const ColumnFamilyOptions& /*cf_opts*/) const override { const ColumnFamilyOptions& /*cf_opts*/) const override {
return Status::NotSupported(); return Status::NotSupported();
} }
std::string GetPrintableTableOptions() const override { return ""; } std::string GetPrintableOptions() const override { return ""; }
Status GetOptionString(const ConfigOptions& /*opts*/,
std::string* /*opt_string*/) const override {
return Status::OK();
}
}; };
class DummyMergeOperator : public MergeOperator { class DummyMergeOperator : public MergeOperator {

@ -35,7 +35,7 @@ class SimCacheTest : public DBTestBase {
options.create_if_missing = true; options.create_if_missing = true;
// options.compression = kNoCompression; // options.compression = kNoCompression;
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics(); options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
return options; return options;
} }
@ -84,7 +84,7 @@ TEST_F(SimCacheTest, SimCache) {
co.metadata_charge_policy = kDontChargeCacheMetadata; co.metadata_charge_policy = kDontChargeCacheMetadata;
std::shared_ptr<SimCache> simCache = NewSimCache(NewLRUCache(co), 20000, 0); std::shared_ptr<SimCache> simCache = NewSimCache(NewLRUCache(co), 20000, 0);
table_options.block_cache = simCache; table_options.block_cache = simCache;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options); Reopen(options);
RecordCacheCounters(options); RecordCacheCounters(options);
@ -151,7 +151,7 @@ TEST_F(SimCacheTest, SimCacheLogging) {
co.metadata_charge_policy = kDontChargeCacheMetadata; co.metadata_charge_policy = kDontChargeCacheMetadata;
std::shared_ptr<SimCache> sim_cache = NewSimCache(NewLRUCache(co), 20000, 0); std::shared_ptr<SimCache> sim_cache = NewSimCache(NewLRUCache(co), 20000, 0);
table_options.block_cache = sim_cache; table_options.block_cache = sim_cache;
options.table_factory.reset(new BlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options); Reopen(options);
int num_block_entries = 20; int num_block_entries = 20;

Loading…
Cancel
Save