From 183cadfc87ee07cbd6e15501092ce4b2ddeed8f3 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Wed, 4 Nov 2015 18:53:30 -0800 Subject: [PATCH] Add OptionsSanityCheckLevel Summary: This patch introduces OptionsSanityCheckLevel internally to enable sanity check rocksdb options. Utilities API will be added in the follow-up diffs. Test Plan: Added more tests in options_test Reviewers: igor, IslamAbdelRahman, sdong, anthony Reviewed By: anthony Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D49515 --- CMakeLists.txt | 1 + src.mk | 3 +- util/options_parser.cc | 142 ++++++++++++++++++++++++----------- util/options_parser.h | 19 +++-- util/options_sanity_check.cc | 38 ++++++++++ util/options_sanity_check.h | 49 ++++++++++++ util/options_test.cc | 128 +++++++++++++++++++++++++++++++ 7 files changed, 330 insertions(+), 50 deletions(-) create mode 100644 util/options_sanity_check.cc create mode 100644 util/options_sanity_check.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d2ea6efb3..f6cf1e26c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -215,6 +215,7 @@ set(SOURCES util/options_builder.cc util/options_helper.cc util/options_parser.cc + util/options_sanity_check.cc util/perf_context.cc util/perf_level.cc util/rate_limiter.cc diff --git a/src.mk b/src.mk index 46083c750..802fc6138 100644 --- a/src.mk +++ b/src.mk @@ -138,10 +138,11 @@ LIB_SOURCES = \ util/memenv.cc \ util/murmurhash.cc \ util/mutable_cf_options.cc \ - util/options_builder.cc \ util/options.cc \ + util/options_builder.cc \ util/options_helper.cc \ util/options_parser.cc \ + util/options_sanity_check.cc \ util/perf_context.cc \ util/perf_level.cc \ util/rate_limiter.cc \ diff --git a/util/options_parser.cc b/util/options_parser.cc index c8439c6fd..fa00a3c3b 100644 --- a/util/options_parser.cc +++ b/util/options_parser.cc @@ -581,7 +581,8 @@ bool AreEqualOptions( Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( const DBOptions& db_opt, const std::vector& cf_names, const std::vector& cf_opts, - const std::string& file_name, Env* env) { + const std::string& file_name, Env* env, + OptionsSanityCheckLevel sanity_check_level) { RocksDBOptionsParser parser; std::unique_ptr seq_file; Status s = parser.Parse(file_name, env); @@ -590,20 +591,28 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( } // Verify DBOptions - s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map()); + s = VerifyDBOptions(db_opt, *parser.db_opt(), parser.db_opt_map(), + sanity_check_level); if (!s.ok()) { return s; } // Verify ColumnFamily Name if (cf_names.size() != parser.cf_names()->size()) { - return Status::Corruption( - "[RocksDBOptionParser Error] The persisted options does not have" - "the same number of column family names as the db instance."); + if (sanity_check_level >= kSanityLevelLooselyCompatible) { + return Status::InvalidArgument( + "[RocksDBOptionParser Error] The persisted options does not have " + "the same number of column family names as the db instance."); + } else if (cf_opts.size() > parser.cf_opts()->size()) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options file has less number of column family " + "names than that of the specified one."); + } } for (size_t i = 0; i < cf_names.size(); ++i) { if (cf_names[i] != parser.cf_names()->at(i)) { - return Status::Corruption( + return Status::InvalidArgument( "[RocksDBOptionParser Error] The persisted options and the db" "instance does not have the same name for column family ", ToString(i)); @@ -612,18 +621,27 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( // Verify Column Family Options if (cf_opts.size() != parser.cf_opts()->size()) { - return Status::Corruption( - "[RocksDBOptionParser Error] The persisted options does not have" - "the same number of column families as the db instance."); + if (sanity_check_level >= kSanityLevelLooselyCompatible) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options does not have the same number of " + "column families as the db instance."); + } else if (cf_opts.size() > parser.cf_opts()->size()) { + return Status::InvalidArgument( + "[RocksDBOptionsParser Error]", + "The persisted options file has less number of column families " + "than that of the specified number."); + } } for (size_t i = 0; i < cf_opts.size(); ++i) { s = VerifyCFOptions(cf_opts[i], parser.cf_opts()->at(i), - &(parser.cf_opt_maps()->at(i))); + &(parser.cf_opt_maps()->at(i)), sanity_check_level); if (!s.ok()) { return s; } s = VerifyTableFactory(cf_opts[i].table_factory.get(), - parser.cf_opts()->at(i).table_factory.get()); + parser.cf_opts()->at(i).table_factory.get(), + sanity_check_level); if (!s.ok()) { return s; } @@ -633,42 +651,75 @@ Status RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( } Status RocksDBOptionsParser::VerifyDBOptions( - const DBOptions& base_opt, const DBOptions& new_opt, - const std::unordered_map* opt_map) { + const DBOptions& base_opt, const DBOptions& persisted_opt, + const std::unordered_map* opt_map, + OptionsSanityCheckLevel sanity_check_level) { for (auto pair : db_options_type_info) { if (pair.second.verification == OptionVerificationType::kDeprecated) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; } - if (!AreEqualOptions(reinterpret_cast(&base_opt), - reinterpret_cast(&new_opt), pair.second, - pair.first, nullptr)) { - return Status::Corruption( - "[RocksDBOptionsParser]: " - "failed the verification on DBOptions::", - pair.first); + if (DBOptionSanityCheckLevel(pair.first) <= sanity_check_level) { + if (!AreEqualOptions(reinterpret_cast(&base_opt), + reinterpret_cast(&persisted_opt), + pair.second, pair.first, nullptr)) { + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + std::string base_value; + std::string persisted_value; + SerializeSingleOptionHelper( + reinterpret_cast(&base_opt) + pair.second.offset, + pair.second.type, &base_value); + SerializeSingleOptionHelper( + reinterpret_cast(&persisted_opt) + pair.second.offset, + pair.second.type, &persisted_value); + snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on DBOptions::%s --- " + "The specified one is %s while the persisted one is %s.\n", + pair.first.c_str(), base_value.c_str(), + persisted_value.c_str()); + return Status::InvalidArgument(Slice(buffer, strlen(buffer))); + } } } return Status::OK(); } Status RocksDBOptionsParser::VerifyCFOptions( - const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt, - const std::unordered_map* new_opt_map) { + const ColumnFamilyOptions& base_opt, + const ColumnFamilyOptions& persisted_opt, + const std::unordered_map* persisted_opt_map, + OptionsSanityCheckLevel sanity_check_level) { for (auto& pair : cf_options_type_info) { if (pair.second.verification == OptionVerificationType::kDeprecated) { // We skip checking deprecated variables as they might // contain random values since they might not be initialized continue; } - if (!AreEqualOptions(reinterpret_cast(&base_opt), - reinterpret_cast(&new_opt), pair.second, - pair.first, new_opt_map)) { - return Status::Corruption( - "[RocksDBOptionsParser]: " - "failed the verification on ColumnFamilyOptions::", - pair.first); + if (CFOptionSanityCheckLevel(pair.first) <= sanity_check_level) { + if (!AreEqualOptions(reinterpret_cast(&base_opt), + reinterpret_cast(&persisted_opt), + pair.second, pair.first, persisted_opt_map)) { + const size_t kBufferSize = 2048; + char buffer[kBufferSize]; + std::string base_value; + std::string persisted_value; + SerializeSingleOptionHelper( + reinterpret_cast(&base_opt) + pair.second.offset, + pair.second.type, &base_value); + SerializeSingleOptionHelper( + reinterpret_cast(&persisted_opt) + pair.second.offset, + pair.second.type, &persisted_value); + snprintf(buffer, sizeof(buffer), + "[RocksDBOptionsParser]: " + "failed the verification on ColumnFamilyOptions::%s --- " + "The specified one is %s while the persisted one is %s.\n", + pair.first.c_str(), base_value.c_str(), + persisted_value.c_str()); + return Status::InvalidArgument(Slice(buffer, sizeof(buffer))); + } } } return Status::OK(); @@ -676,8 +727,10 @@ Status RocksDBOptionsParser::VerifyCFOptions( Status RocksDBOptionsParser::VerifyBlockBasedTableFactory( const BlockBasedTableFactory* base_tf, - const BlockBasedTableFactory* file_tf) { - if ((base_tf != nullptr) != (file_tf != nullptr)) { + const BlockBasedTableFactory* file_tf, + OptionsSanityCheckLevel sanity_check_level) { + if ((base_tf != nullptr) != (file_tf != nullptr) && + sanity_check_level > kSanityLevelNone) { return Status::Corruption( "[RocksDBOptionsParser]: Inconsistent TableFactory class type"); } @@ -694,29 +747,34 @@ Status RocksDBOptionsParser::VerifyBlockBasedTableFactory( // contain random values since they might not be initialized continue; } - if (!AreEqualOptions(reinterpret_cast(&base_opt), - reinterpret_cast(&file_opt), pair.second, - pair.first, nullptr)) { - return Status::Corruption( - "[RocksDBOptionsParser]: " - "failed the verification on BlockBasedTableOptions::", - pair.first); + if (BBTOptionSanityCheckLevel(pair.first) <= sanity_check_level) { + if (!AreEqualOptions(reinterpret_cast(&base_opt), + reinterpret_cast(&file_opt), + pair.second, pair.first, nullptr)) { + return Status::Corruption( + "[RocksDBOptionsParser]: " + "failed the verification on BlockBasedTableOptions::", + pair.first); + } } } return Status::OK(); } -Status RocksDBOptionsParser::VerifyTableFactory(const TableFactory* base_tf, - const TableFactory* file_tf) { +Status RocksDBOptionsParser::VerifyTableFactory( + const TableFactory* base_tf, const TableFactory* file_tf, + OptionsSanityCheckLevel sanity_check_level) { if (base_tf && file_tf) { - if (base_tf->Name() != file_tf->Name()) { + if (sanity_check_level > kSanityLevelNone && + base_tf->Name() != file_tf->Name()) { return Status::Corruption( "[RocksDBOptionsParser]: " "failed the verification on TableFactory->Name()"); } auto s = VerifyBlockBasedTableFactory( dynamic_cast(base_tf), - dynamic_cast(file_tf)); + dynamic_cast(file_tf), + sanity_check_level); if (!s.ok()) { return s; } diff --git a/util/options_parser.h b/util/options_parser.h index 4ceae7254..e5dfba17a 100644 --- a/util/options_parser.h +++ b/util/options_parser.h @@ -11,6 +11,8 @@ #include "rocksdb/env.h" #include "rocksdb/options.h" +#include "util/options_helper.h" +#include "util/options_sanity_check.h" #include "table/block_based_table_factory.h" namespace rocksdb { @@ -65,24 +67,27 @@ class RocksDBOptionsParser { static Status VerifyRocksDBOptionsFromFile( const DBOptions& db_opt, const std::vector& cf_names, const std::vector& cf_opts, - const std::string& file_name, Env* env); + const std::string& file_name, Env* env, + OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); static Status VerifyDBOptions( const DBOptions& base_opt, const DBOptions& new_opt, - const std::unordered_map* new_opt_map = - nullptr); + const std::unordered_map* new_opt_map = nullptr, + OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); static Status VerifyCFOptions( const ColumnFamilyOptions& base_opt, const ColumnFamilyOptions& new_opt, - const std::unordered_map* new_opt_map = - nullptr); + const std::unordered_map* new_opt_map = nullptr, + OptionsSanityCheckLevel sanity_check_level = kSanityLevelExactMatch); static Status VerifyTableFactory(const TableFactory* base_tf, - const TableFactory* file_tf); + const TableFactory* file_tf, + OptionsSanityCheckLevel sanity_check_level); static Status VerifyBlockBasedTableFactory( const BlockBasedTableFactory* base_tf, - const BlockBasedTableFactory* file_tf); + const BlockBasedTableFactory* file_tf, + OptionsSanityCheckLevel sanity_check_level); static Status ExtraParserCheck(const RocksDBOptionsParser& input_parser); diff --git a/util/options_sanity_check.cc b/util/options_sanity_check.cc new file mode 100644 index 000000000..a84031bf9 --- /dev/null +++ b/util/options_sanity_check.cc @@ -0,0 +1,38 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef ROCKSDB_LITE + +#include "util/options_sanity_check.h" + +namespace rocksdb { + +namespace { +OptionsSanityCheckLevel SanityCheckLevelHelper( + const std::unordered_map& smap, + const std::string& name) { + auto iter = smap.find(name); + return iter != smap.end() ? iter->second : kSanityLevelExactMatch; +} +} + +OptionsSanityCheckLevel DBOptionSanityCheckLevel( + const std::string& option_name) { + return SanityCheckLevelHelper(sanity_level_db_options, option_name); +} + +OptionsSanityCheckLevel CFOptionSanityCheckLevel( + const std::string& option_name) { + return SanityCheckLevelHelper(sanity_level_cf_options, option_name); +} + +OptionsSanityCheckLevel BBTOptionSanityCheckLevel( + const std::string& option_name) { + return SanityCheckLevelHelper(sanity_level_bbt_options, option_name); +} + +} // namespace rocksdb + +#endif // !ROCKSDB_LITE diff --git a/util/options_sanity_check.h b/util/options_sanity_check.h new file mode 100644 index 000000000..eba26462e --- /dev/null +++ b/util/options_sanity_check.h @@ -0,0 +1,49 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#pragma once + +#include +#include + +#ifndef ROCKSDB_LITE +namespace rocksdb { +// This enum defines the RocksDB options sanity level. +enum OptionsSanityCheckLevel : unsigned char { + // Performs no sanity check at all. + kSanityLevelNone = 0x00, + // Performs minimum check to ensure the RocksDB instance can be + // opened without corrupting / mis-interpreting the data. + kSanityLevelLooselyCompatible = 0x01, + // Perform exact match sanity check. + kSanityLevelExactMatch = 0xFF, +}; + +// The sanity check level for DB options +static const std::unordered_map + sanity_level_db_options = {}; + +// The sanity check level for column-family options +static const std::unordered_map + sanity_level_cf_options = { + {"comparator", kSanityLevelLooselyCompatible}, + {"prefix_extractor", kSanityLevelLooselyCompatible}, + {"table_factory", kSanityLevelLooselyCompatible}, + {"merge_operator", kSanityLevelLooselyCompatible}}; + +// The sanity check level for block-based table options +static const std::unordered_map + sanity_level_bbt_options = {}; + +OptionsSanityCheckLevel DBOptionSanityCheckLevel( + const std::string& options_name); +OptionsSanityCheckLevel CFOptionSanityCheckLevel( + const std::string& options_name); +OptionsSanityCheckLevel BBTOptionSanityCheckLevel( + const std::string& options_name); + +} // namespace rocksdb + +#endif // !ROCKSDB_LITE diff --git a/util/options_test.cc b/util/options_test.cc index 384e9e5aa..7940fde18 100644 --- a/util/options_test.cc +++ b/util/options_test.cc @@ -26,6 +26,7 @@ #include "table/plain_table_factory.h" #include "util/options_helper.h" #include "util/options_parser.h" +#include "util/options_sanity_check.h" #include "util/random.h" #include "util/testharness.h" #include "util/testutil.h" @@ -1644,6 +1645,133 @@ TEST_F(OptionsParserTest, DifferentDefault) { ASSERT_OK(parser.Parse(kOptionsFileName, env_.get())); } +class OptionsSanityCheckTest : public OptionsParserTest { + public: + OptionsSanityCheckTest() {} + + protected: + Status SanityCheckCFOptions(const ColumnFamilyOptions& cf_opts, + OptionsSanityCheckLevel level) { + return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( + DBOptions(), {"default"}, {cf_opts}, kOptionsFileName, env_.get(), + level); + } + + Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) { + Status s = env_->DeleteFile(kOptionsFileName); + if (!s.ok()) { + return s; + } + return PersistRocksDBOptions(DBOptions(), {"default"}, {cf_opts}, + kOptionsFileName, env_.get()); + } + + const std::string kOptionsFileName = "OPTIONS"; +}; + +TEST_F(OptionsSanityCheckTest, SanityCheck) { + ColumnFamilyOptions opts; + Random rnd(301); + + // default ColumnFamilyOptions + { + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + } + + // prefix_extractor + { + // change the prefix extractor and expect only pass when + // sanity-level == kSanityLevelNone + opts.prefix_extractor.reset(NewCappedPrefixTransform(10)); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + + // use same prefix extractor but with different parameter + opts.prefix_extractor.reset(NewCappedPrefixTransform(15)); + // expect pass only in kSanityLevelNone + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // repeat the test with FixedPrefixTransform + opts.prefix_extractor.reset(NewFixedPrefixTransform(10)); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change of prefix_extractor + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + + // use same prefix extractor but with different parameter + opts.prefix_extractor.reset(NewFixedPrefixTransform(15)); + // expect pass only in kSanityLevelNone + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + } + + // table_factory + { + for (int tb = 2; tb >= 0; --tb) { + // change the table factory + opts.table_factory.reset(RandomTableFactory(&rnd, tb)); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + } + } + + // merge_operator + { + for (int test = 0; test < 5; ++test) { + // change the merge operator + opts.merge_operator.reset(RandomMergeOperator(&rnd)); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelNone)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + } + } + + // compaction_filter + { + for (int test = 0; test < 5; ++test) { + // change the compaction filter + opts.compaction_filter = RandomCompactionFilter(&rnd); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + delete opts.compaction_filter; + opts.compaction_filter = nullptr; + } + } + + // compaction_filter_factory + { + for (int test = 0; test < 5; ++test) { + // change the compaction filter factory + opts.compaction_filter_factory.reset(RandomCompactionFilterFactory(&rnd)); + ASSERT_NOK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelLooselyCompatible)); + + // persist the change + ASSERT_OK(PersistCFOptions(opts)); + ASSERT_OK(SanityCheckCFOptions(opts, kSanityLevelExactMatch)); + } + } +} + namespace { bool IsEscapedString(const std::string& str) { for (size_t i = 0; i < str.size(); ++i) {