From f1ed1701058a0832076574f6a514df227897213d Mon Sep 17 00:00:00 2001 From: sdong Date: Wed, 20 Jan 2016 18:15:36 -0800 Subject: [PATCH] Add tests to make sure new DB or ColumnFamily options are settable through string Summary: Add a test to fail if someone adds a DB options. Test Plan: Run the test, run the test with valgrind. Add an option to DB option in the middle or in the end and make sure it fails. Reviewers: yhchiang, anthony, IslamAbdelRahman, kradhakrishnan, rven, andrewkr Reviewed By: andrewkr Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D53097 --- util/options_test.cc | 380 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 350 insertions(+), 30 deletions(-) diff --git a/util/options_test.cc b/util/options_test.cc index 19e48b350..09ecbea03 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -1475,39 +1475,28 @@ TEST_F(OptionsParserTest, EscapeOptionString) { "Escape \\# and"); } -// Only run OptionsParserTest.BlockBasedTableOptionsAdded on limited platforms -// as it depends on behavior of compilers. +// Only run the tests to verify new fields in options are settable through +// string on limited platforms as it depends on behavior of compilers. #ifdef OS_LINUX #ifndef __clang__ const char kSpecialChar = 'R'; -// Items in the form of . Need to be in ascending order -// and not overlapping. Need to updated if new pointer-option is added. -const std::vector> kBbtoBlacklist = { - {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), - sizeof(std::shared_ptr)}, - {offsetof(struct BlockBasedTableOptions, block_cache), - sizeof(std::shared_ptr)}, - {offsetof(struct BlockBasedTableOptions, block_cache_compressed), - sizeof(std::shared_ptr)}, - {offsetof(struct BlockBasedTableOptions, filter_policy), - sizeof(std::shared_ptr)}, -}; +typedef std::vector> OffsetGap; -void FillWithSpecialChar(char* start_ptr) { +void FillWithSpecialChar(char* start_ptr, size_t total_size, + const OffsetGap& blacklist) { size_t offset = 0; - for (auto& pair : kBbtoBlacklist) { + for (auto& pair : blacklist) { std::memset(start_ptr + offset, kSpecialChar, pair.first - offset); offset = pair.first + pair.second; } - std::memset(start_ptr + offset, kSpecialChar, - sizeof(BlockBasedTableOptions) - offset); + std::memset(start_ptr + offset, kSpecialChar, total_size - offset); } -int NumUnsetBytes(char* start_ptr) { +int NumUnsetBytes(char* start_ptr, size_t total_size, + const OffsetGap& blacklist) { int total_unset_bytes_base = 0; - size_t offset = 0; - for (auto& pair : kBbtoBlacklist) { + for (auto& pair : blacklist) { for (char* ptr = start_ptr + offset; ptr < start_ptr + pair.first; ptr++) { if (*ptr == kSpecialChar) { total_unset_bytes_base++; @@ -1515,8 +1504,7 @@ int NumUnsetBytes(char* start_ptr) { offset = pair.first + pair.second; } } - for (char* ptr = start_ptr + offset; - ptr < start_ptr + sizeof(BlockBasedTableOptions); ptr++) { + for (char* ptr = start_ptr + offset; ptr < start_ptr + total_size; ptr++) { if (*ptr == kSpecialChar) { total_unset_bytes_base++; } @@ -1524,7 +1512,28 @@ int NumUnsetBytes(char* start_ptr) { return total_unset_bytes_base; } +// If the test fails, likely a new option is added to BlockBasedTableOptions +// but it cannot be set through GetBlockBasedTableOptionsFromString(), or the +// test is not updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetBlockBasedTableOptionsFromString() and add the option to the input string +// passed to the GetBlockBasedTableOptionsFromString() in this test. +// If it is a complicated type, you also need to add the field to +// kBbtoBlacklist, and maybe add customized verification for it. TEST_F(OptionsParserTest, BlockBasedTableOptionsAllFieldsSettable) { + // Items in the form of . Need to be in ascending order + // and not overlapping. Need to updated if new pointer-option is added. + const OffsetGap kBbtoBlacklist = { + {offsetof(struct BlockBasedTableOptions, flush_block_policy_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct BlockBasedTableOptions, block_cache), + sizeof(std::shared_ptr)}, + {offsetof(struct BlockBasedTableOptions, block_cache_compressed), + sizeof(std::shared_ptr)}, + {offsetof(struct BlockBasedTableOptions, filter_policy), + sizeof(std::shared_ptr)}, + }; + // In this test, we catch a new option of BlockBasedTableOptions that is not // settable through GetBlockBasedTableOptionsFromString(). // We count padding bytes of the option struct, and assert it to be the same @@ -1537,28 +1546,31 @@ TEST_F(OptionsParserTest, BlockBasedTableOptionsAllFieldsSettable) { // copy a well constructed struct to this memory and see how many special // bytes left. BlockBasedTableOptions* bbto = new (bbto_ptr) BlockBasedTableOptions(); - FillWithSpecialChar(bbto_ptr); + FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); // It based on the behavior of compiler that padding bytes are not changed // when copying the struct. It's prone to failure when compiler behavior // changes. We verify there is unset bytes to detect the case. *bbto = BlockBasedTableOptions(); - int unset_bytes_base = NumUnsetBytes(bbto_ptr); + int unset_bytes_base = + NumUnsetBytes(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); ASSERT_GT(unset_bytes_base, 0); bbto->~BlockBasedTableOptions(); // Construct the base option passed into // GetBlockBasedTableOptionsFromString(). - FillWithSpecialChar(bbto_ptr); + bbto = new (bbto_ptr) BlockBasedTableOptions(); + FillWithSpecialChar(bbto_ptr, sizeof(BlockBasedTableOptions), kBbtoBlacklist); // This option is not setable: bbto->use_delta_encoding = true; char* new_bbto_ptr = new char[sizeof(BlockBasedTableOptions)]; BlockBasedTableOptions* new_bbto = new (new_bbto_ptr) BlockBasedTableOptions(); - FillWithSpecialChar(new_bbto_ptr); + FillWithSpecialChar(new_bbto_ptr, sizeof(BlockBasedTableOptions), + kBbtoBlacklist); // Need to update the option string if a new option is added. - GetBlockBasedTableOptionsFromString( + ASSERT_OK(GetBlockBasedTableOptionsFromString( *bbto, "cache_index_and_filter_blocks=1;index_type=kHashSearch;" "checksum=kxxHash;hash_index_allow_collision=1;no_block_cache=1;" @@ -1567,9 +1579,11 @@ TEST_F(OptionsParserTest, BlockBasedTableOptionsAllFieldsSettable) { "filter_policy=bloomfilter:4:true;whole_key_filtering=1;" "skip_table_builder_flush=1;format_version=1;" "hash_index_allow_collision=false;", - new_bbto); + new_bbto)); - ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_bbto_ptr)); + ASSERT_EQ(unset_bytes_base, + NumUnsetBytes(new_bbto_ptr, sizeof(BlockBasedTableOptions), + kBbtoBlacklist)); ASSERT_TRUE(new_bbto->block_cache.get() != nullptr); ASSERT_TRUE(new_bbto->block_cache_compressed.get() != nullptr); @@ -1581,6 +1595,312 @@ TEST_F(OptionsParserTest, BlockBasedTableOptionsAllFieldsSettable) { delete[] bbto_ptr; delete[] new_bbto_ptr; } + +// If the test fails, likely a new option is added to DBOptions +// but it cannot be set through GetDBOptionsFromString(), or the test is not +// updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetDBOptionsFromString() and add the option to the input string passed to +// DBOptionsFromString()in this test. +// If it is a complicated type, you also need to add the field to +// kDBOptionsBlacklist, and maybe add customized verification for it. +TEST_F(OptionsParserTest, DBOptionsAllFieldsSettable) { + const OffsetGap kDBOptionsBlacklist = { + {offsetof(struct DBOptions, env), sizeof(Env*)}, + {offsetof(struct DBOptions, rate_limiter), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, delete_scheduler), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, info_log), sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, statistics), + sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, db_paths), sizeof(std::vector)}, + {offsetof(struct DBOptions, db_log_dir), sizeof(std::string)}, + {offsetof(struct DBOptions, wal_dir), sizeof(std::string)}, + {offsetof(struct DBOptions, listeners), + sizeof(std::vector>)}, + {offsetof(struct DBOptions, row_cache), sizeof(std::shared_ptr)}, + {offsetof(struct DBOptions, wal_filter), sizeof(const WalFilter*)}, + }; + + char* options_ptr = new char[sizeof(DBOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + DBOptions* options = new (options_ptr) DBOptions(); + FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *options = DBOptions(); + int unset_bytes_base = + NumUnsetBytes(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + ASSERT_GT(unset_bytes_base, 0); + options->~DBOptions(); + + options = new (options_ptr) DBOptions(); + FillWithSpecialChar(options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + + // Following options are not settable through GetDBOptionsFromString(): + options->fail_if_options_file_error = false; + options->allow_concurrent_memtable_write = false; + options->wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery; + options->enable_write_thread_adaptive_yield = true; + options->write_thread_slow_yield_usec = true; + options->write_thread_max_yield_usec = 1000u; + options->access_hint_on_compaction_start = DBOptions::AccessHint::NONE; + options->info_log_level = InfoLogLevel::DEBUG_LEVEL; + + char* new_options_ptr = new char[sizeof(DBOptions)]; + DBOptions* new_options = new (new_options_ptr) DBOptions(); + FillWithSpecialChar(new_options_ptr, sizeof(DBOptions), kDBOptionsBlacklist); + + // Need to update the option string if a new option is added. + ASSERT_OK( + GetDBOptionsFromString(*options, + "wal_bytes_per_sync=4295048118;" + "delete_obsolete_files_period_micros=4294967758;" + "WAL_ttl_seconds=4295008036;" + "WAL_size_limit_MB=4295036161;" + "wal_dir=path/to/wal_dir;" + "db_write_buffer_size=2587;" + "max_subcompactions=64330;" + "table_cache_numshardbits=28;" + "max_open_files=72;" + "max_file_opening_threads=35;" + "max_background_compactions=33;" + "use_fsync=true;" + "use_adaptive_mutex=false;" + "max_total_wal_size=4295005604;" + "compaction_readahead_size=0;" + "new_table_reader_for_compaction_inputs=false;" + "keep_log_file_num=4890;" + "skip_stats_update_on_db_open=false;" + "max_manifest_file_size=4295009941;" + "db_log_dir=path/to/db_log_dir;" + "skip_log_error_on_recovery=true;" + "writable_file_max_buffer_size=1048576;" + "paranoid_checks=true;" + "is_fd_close_on_exec=false;" + "bytes_per_sync=4295013613;" + "enable_thread_tracking=false;" + "disable_data_sync=false;" + "recycle_log_file_num=0;" + "disableDataSync=false;" + "create_missing_column_families=true;" + "log_file_time_to_roll=3097;" + "max_background_flushes=35;" + "create_if_missing=false;" + "error_if_exists=true;" + "allow_os_buffer=false;" + "delayed_write_rate=4294976214;" + "manifest_preallocation_size=1222;" + "allow_mmap_writes=false;" + "stats_dump_period_sec=70127;" + "allow_fallocate=true;" + "allow_mmap_reads=false;" + "max_log_file_size=4607;" + "random_access_max_buffer_size=1048576;" + "advise_random_on_open=true;" + "wal_bytes_per_sync=4295048118;" + "delete_obsolete_files_period_micros=4294967758;" + "WAL_ttl_seconds=4295008036;" + "WAL_size_limit_MB=4295036161;" + "wal_dir=path/to/wal_dir;" + "db_write_buffer_size=2587;" + "max_subcompactions=64330;" + "table_cache_numshardbits=28;" + "max_open_files=72;" + "max_file_opening_threads=35;" + "max_background_compactions=33;" + "use_fsync=true;" + "use_adaptive_mutex=false;" + "max_total_wal_size=4295005604;" + "compaction_readahead_size=0;" + "new_table_reader_for_compaction_inputs=false;" + "keep_log_file_num=4890;" + "skip_stats_update_on_db_open=false;" + "max_manifest_file_size=4295009941;" + "db_log_dir=path/to/db_log_dir;" + "skip_log_error_on_recovery=true;" + "writable_file_max_buffer_size=1048576;" + "paranoid_checks=true;" + "is_fd_close_on_exec=false;" + "bytes_per_sync=4295013613;" + "enable_thread_tracking=false;" + "disable_data_sync=false;" + "recycle_log_file_num=0;" + "disableDataSync=false;" + "create_missing_column_families=true;" + "log_file_time_to_roll=3097;" + "max_background_flushes=35;" + "create_if_missing=false;" + "error_if_exists=true;" + "allow_os_buffer=false;" + "delayed_write_rate=4294976214;" + "manifest_preallocation_size=1222;" + "allow_mmap_writes=false;" + "stats_dump_period_sec=70127;" + "allow_fallocate=true;" + "allow_mmap_reads=false;" + "max_log_file_size=4607;" + "random_access_max_buffer_size=1048576;" + "advise_random_on_open=true;", + new_options)); + + ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), + kDBOptionsBlacklist)); + + options->~DBOptions(); + new_options->~DBOptions(); + + delete[] options_ptr; + delete[] new_options_ptr; +} + +// If the test fails, likely a new option is added to ColumnFamilyOptions +// but it cannot be set through GetColumnFamilyOptionsFromString(), or the +// test is not updated accordingly. +// After adding an option, we need to make sure it is settable by +// GetColumnFamilyOptionsFromString() and add the option to the input +// string passed to GetColumnFamilyOptionsFromString()in this test. +// If it is a complicated type, you also need to add the field to +// kColumnFamilyOptionsBlacklist, and maybe add customized verification +// for it. +TEST_F(OptionsParserTest, ColumnFamilyOptionsAllFieldsSettable) { + const OffsetGap kColumnFamilyOptionsBlacklist = { + {offsetof(struct ColumnFamilyOptions, comparator), sizeof(Comparator*)}, + {offsetof(struct ColumnFamilyOptions, merge_operator), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, compaction_filter), + sizeof(const CompactionFilter*)}, + {offsetof(struct ColumnFamilyOptions, compaction_filter_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, compression_per_level), + sizeof(std::vector)}, + {offsetof(struct ColumnFamilyOptions, prefix_extractor), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, + max_bytes_for_level_multiplier_additional), + sizeof(std::vector)}, + {offsetof(struct ColumnFamilyOptions, memtable_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, table_factory), + sizeof(std::shared_ptr)}, + {offsetof(struct ColumnFamilyOptions, + table_properties_collector_factories), + sizeof(ColumnFamilyOptions::TablePropertiesCollectorFactories)}, + {offsetof(struct ColumnFamilyOptions, inplace_callback), + sizeof(UpdateStatus (*)(char*, uint32_t*, Slice, std::string*))}, + }; + + char* options_ptr = new char[sizeof(ColumnFamilyOptions)]; + + // Count padding bytes by setting all bytes in the memory to a special char, + // copy a well constructed struct to this memory and see how many special + // bytes left. + ColumnFamilyOptions* options = new (options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + // It based on the behavior of compiler that padding bytes are not changed + // when copying the struct. It's prone to failure when compiler behavior + // changes. We verify there is unset bytes to detect the case. + *options = ColumnFamilyOptions(); + + // Deprecatd option which is not initialized. Need to set it to avoid + // Valgrind error + options->max_mem_compaction_level = 0; + + int unset_bytes_base = NumUnsetBytes(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + ASSERT_GT(unset_bytes_base, 0); + options->~ColumnFamilyOptions(); + + options = new (options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + + // Following options are not settable through + // GetColumnFamilyOptionsFromString(): + options->rate_limit_delay_max_milliseconds = 33; + options->compaction_pri = CompactionPri::kOldestSmallestSeqFirst; + options->compaction_options_universal = CompactionOptionsUniversal(); + options->compression_opts = CompressionOptions(); + options->hard_rate_limit = 0; + options->soft_rate_limit = 0; + options->compaction_options_fifo = CompactionOptionsFIFO(); + options->max_mem_compaction_level = 0; + + char* new_options_ptr = new char[sizeof(ColumnFamilyOptions)]; + ColumnFamilyOptions* new_options = + new (new_options_ptr) ColumnFamilyOptions(); + FillWithSpecialChar(new_options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist); + + // Need to update the option string if a new option is added. + ASSERT_OK(GetColumnFamilyOptionsFromString( + *options, + "compaction_filter_factory=mpudlojcujCompactionFilterFactory;" + "table_factory=PlainTable;" + "prefix_extractor=rocksdb.CappedPrefix.13;" + "comparator=leveldb.BytewiseComparator;" + "compression_per_level=kBZip2Compression:kBZip2Compression:" + "kBZip2Compression:kNoCompression:kZlibCompression:kBZip2Compression:" + "kSnappyCompression;" + "max_bytes_for_level_base=986;" + "bloom_locality=8016;" + "target_file_size_base=4294976376;" + "memtable_prefix_bloom_huge_page_tlb_size=2557;" + "max_successive_merges=5497;" + "max_sequential_skip_in_iterations=4294971408;" + "arena_block_size=1893;" + "target_file_size_multiplier=35;" + "source_compaction_factor=54;" + "min_write_buffer_number_to_merge=9;" + "max_write_buffer_number=84;" + "write_buffer_size=1653;" + "max_grandparent_overlap_factor=64;" + "max_bytes_for_level_multiplier=60;" + "memtable_factory=SkipListFactory;" + "compression=kNoCompression;" + "min_partial_merge_operands=7576;" + "level0_stop_writes_trigger=33;" + "num_levels=99;" + "level0_slowdown_writes_trigger=22;" + "level0_file_num_compaction_trigger=14;" + "expanded_compaction_factor=34;" + "compaction_filter=urxcqstuwnCompactionFilter;" + "soft_rate_limit=530.615385;" + "soft_pending_compaction_bytes_limit=0;" + "max_write_buffer_number_to_maintain=84;" + "verify_checksums_in_compaction=false;" + "merge_operator=aabcxehazrMergeOperator;" + "memtable_prefix_bloom_bits=4642;" + "paranoid_file_checks=true;" + "inplace_update_num_locks=7429;" + "optimize_filters_for_hits=false;" + "level_compaction_dynamic_level_bytes=false;" + "inplace_update_support=false;" + "compaction_style=kCompactionStyleFIFO;" + "memtable_prefix_bloom_probes=2511;" + "purge_redundant_kvs_while_flush=true;" + "filter_deletes=false;" + "hard_pending_compaction_bytes_limit=0;" + "disable_auto_compactions=false;" + "compaction_measure_io_stats=true;", + new_options)); + + ASSERT_EQ(unset_bytes_base, + NumUnsetBytes(new_options_ptr, sizeof(ColumnFamilyOptions), + kColumnFamilyOptionsBlacklist)); + + options->~ColumnFamilyOptions(); + new_options->~ColumnFamilyOptions(); + + delete[] options_ptr; + delete[] new_options_ptr; +} #endif // !__clang__ #endif // OS_LINUX #endif // !ROCKSDB_LITE