Add CheckOptionsCompatibility() API to options_util

Summary:
Add CheckOptionsCompatibility() API to options_util that returns
Status::OK if the input DBOptions and ColumnFamilyDescriptors
are compatible with the latest options stored in the specified DB path.

Test Plan: Added tests in options_util_test

Reviewers: igor, anthony, IslamAbdelRahman, rven, sdong

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D50649
main
Yueh-Hsuan Chiang 9 years ago
parent 2391b459b8
commit d781da8164
  1. 1
      HISTORY.md
  2. 15
      include/rocksdb/utilities/options_util.h
  3. 23
      utilities/options/options_util.cc
  4. 193
      utilities/options/options_util_test.cc

@ -7,6 +7,7 @@
* Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances. * Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances.
* RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions. * RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions.
* Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h. This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance. * Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h. This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance.
* Introduce CheckOptionsCompatibility() in rocksdb/utilities/options_util.h. This function checks whether the input set of options is able to open the specified DB successfully.
### Public API Changes ### Public API Changes
* CompactionFilter::Context includes information of Column Family ID * CompactionFilter::Context includes information of Column Family ID
* The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower. * The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower.

@ -61,5 +61,20 @@ Status LoadOptionsFromFile(const std::string& options_file_name, Env* env,
Status GetLatestOptionsFileName(const std::string& dbpath, Env* env, Status GetLatestOptionsFileName(const std::string& dbpath, Env* env,
std::string* options_file_name); std::string* options_file_name);
// Returns Status::OK if the input DBOptions and ColumnFamilyDescriptors
// are compatible with the latest options stored in the specified DB path.
//
// If the return status is non-ok, it means the specified RocksDB instance
// might not be correctly opened with the input set of options. Currently,
// changing one of the following options will fail the compatibility check:
//
// * comparator
// * prefix_extractor
// * table_factory
// * merge_operator
Status CheckOptionsCompatibility(
const std::string& dbpath, Env* env, const DBOptions& db_options,
const std::vector<ColumnFamilyDescriptor>& cf_descs);
} // namespace rocksdb } // namespace rocksdb
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE

@ -72,5 +72,28 @@ Status LoadLatestOptions(const std::string& dbpath, Env* env,
db_options, cf_descs); db_options, cf_descs);
} }
Status CheckOptionsCompatibility(
const std::string& dbpath, Env* env, const DBOptions& db_options,
const std::vector<ColumnFamilyDescriptor>& cf_descs) {
std::string options_file_name;
Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name);
if (!s.ok()) {
return s;
}
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
for (const auto& cf_desc : cf_descs) {
cf_names.push_back(cf_desc.name);
cf_opts.push_back(cf_desc.options);
}
const OptionsSanityCheckLevel kDefaultLevel = kSanityLevelLooselyCompatible;
return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
db_options, cf_names, cf_opts, dbpath + "/" + options_file_name, env,
kDefaultLevel);
}
} // namespace rocksdb } // namespace rocksdb
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE

@ -13,6 +13,8 @@
#include <cctype> #include <cctype>
#include <unordered_map> #include <unordered_map>
#include "rocksdb/db.h"
#include "rocksdb/table.h"
#include "rocksdb/utilities/options_util.h" #include "rocksdb/utilities/options_util.h"
#include "util/options_parser.h" #include "util/options_parser.h"
#include "util/random.h" #include "util/random.h"
@ -30,10 +32,15 @@ DEFINE_bool(enable_print, false, "Print options generated to console.");
namespace rocksdb { namespace rocksdb {
class OptionsUtilTest : public testing::Test { class OptionsUtilTest : public testing::Test {
public: public:
OptionsUtilTest() { env_.reset(new test::StringEnv(Env::Default())); } OptionsUtilTest() : rnd_(0xFB) {
env_.reset(new test::StringEnv(Env::Default()));
dbname_ = test::TmpDir() + "/options_util_test";
}
protected: protected:
std::unique_ptr<test::StringEnv> env_; std::unique_ptr<test::StringEnv> env_;
std::string dbname_;
Random rnd_;
}; };
bool IsBlockBasedTableFactory(TableFactory* tf) { bool IsBlockBasedTableFactory(TableFactory* tf) {
@ -42,17 +49,16 @@ bool IsBlockBasedTableFactory(TableFactory* tf) {
TEST_F(OptionsUtilTest, SaveAndLoad) { TEST_F(OptionsUtilTest, SaveAndLoad) {
const size_t kCFCount = 5; const size_t kCFCount = 5;
Random rnd(0xFB);
DBOptions db_opt; DBOptions db_opt;
std::vector<std::string> cf_names; std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts; std::vector<ColumnFamilyOptions> cf_opts;
test::RandomInitDBOptions(&db_opt, &rnd); test::RandomInitDBOptions(&db_opt, &rnd_);
for (size_t i = 0; i < kCFCount; ++i) { for (size_t i = 0; i < kCFCount; ++i) {
cf_names.push_back(i == 0 ? kDefaultColumnFamilyName cf_names.push_back(i == 0 ? kDefaultColumnFamilyName
: test::RandomName(&rnd, 10)); : test::RandomName(&rnd_, 10));
cf_opts.emplace_back(); cf_opts.emplace_back();
test::RandomInitCFOptions(&cf_opts.back(), &rnd); test::RandomInitCFOptions(&cf_opts.back(), &rnd_);
} }
const std::string kFileName = "OPTIONS-123456"; const std::string kFileName = "OPTIONS-123456";
@ -64,7 +70,7 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
&loaded_cf_descs)); &loaded_cf_descs));
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt));
test::RandomInitDBOptions(&db_opt, &rnd); test::RandomInitDBOptions(&db_opt, &rnd_);
ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt));
for (size_t i = 0; i < kCFCount; ++i) { for (size_t i = 0; i < kCFCount; ++i) {
@ -76,7 +82,7 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
cf_opts[i].table_factory.get(), cf_opts[i].table_factory.get(),
loaded_cf_descs[i].options.table_factory.get())); loaded_cf_descs[i].options.table_factory.get()));
} }
test::RandomInitCFOptions(&cf_opts[i], &rnd); test::RandomInitCFOptions(&cf_opts[i], &rnd_);
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
cf_opts[i], loaded_cf_descs[i].options)); cf_opts[i], loaded_cf_descs[i].options));
} }
@ -88,6 +94,179 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
} }
} }
namespace {
class DummyTableFactory : public TableFactory {
public:
DummyTableFactory() {}
virtual ~DummyTableFactory() {}
virtual const char* Name() const { return "DummyTableFactory"; }
virtual Status NewTableReader(const TableReaderOptions& table_reader_options,
unique_ptr<RandomAccessFileReader>&& file,
uint64_t file_size,
unique_ptr<TableReader>* table_reader) const {
return Status::NotSupported();
}
virtual TableBuilder* NewTableBuilder(
const TableBuilderOptions& table_builder_options,
uint32_t column_family_id, WritableFileWriter* file) const {
return nullptr;
}
virtual Status SanitizeOptions(const DBOptions& db_opts,
const ColumnFamilyOptions& cf_opts) const {
return Status::NotSupported();
}
virtual std::string GetPrintableTableOptions() const { return ""; }
};
class DummyMergeOperator : public MergeOperator {
public:
DummyMergeOperator() {}
virtual ~DummyMergeOperator() {}
virtual bool FullMerge(const Slice& key, const Slice* existing_value,
const std::deque<std::string>& operand_list,
std::string* new_value, Logger* logger) const {
return false;
}
virtual bool PartialMergeMulti(const Slice& key,
const std::deque<Slice>& operand_list,
std::string* new_value, Logger* logger) const {
return false;
}
virtual const char* Name() const { return "DummyMergeOperator"; }
};
class DummySliceTransform : public SliceTransform {
public:
DummySliceTransform() {}
virtual ~DummySliceTransform() {}
// Return the name of this transformation.
virtual const char* Name() const { return "DummySliceTransform"; }
// transform a src in domain to a dst in the range
virtual Slice Transform(const Slice& src) const { return src; }
// determine whether this is a valid src upon the function applies
virtual bool InDomain(const Slice& src) const { return false; }
// determine whether dst=Transform(src) for some src
virtual bool InRange(const Slice& dst) const { return false; }
};
} // namespace
TEST_F(OptionsUtilTest, SanityCheck) {
DBOptions db_opt;
std::vector<ColumnFamilyDescriptor> cf_descs;
const size_t kCFCount = 5;
for (size_t i = 0; i < kCFCount; ++i) {
cf_descs.emplace_back();
cf_descs.back().name =
(i == 0) ? kDefaultColumnFamilyName : test::RandomName(&rnd_, 10);
cf_descs.back().options.table_factory.reset(NewBlockBasedTableFactory());
cf_descs.back().options.prefix_extractor.reset(
test::RandomSliceTransform(&rnd_));
cf_descs.back().options.merge_operator.reset(
test::RandomMergeOperator(&rnd_));
}
db_opt.create_missing_column_families = true;
db_opt.create_if_missing = true;
DestroyDB(dbname_, Options(db_opt, cf_descs[0].options));
DB* db;
std::vector<ColumnFamilyHandle*> handles;
// open and persist the options
ASSERT_OK(DB::Open(db_opt, dbname_, cf_descs, &handles, &db));
// close the db
for (auto* handle : handles) {
delete handle;
}
delete db;
// perform sanity check
ASSERT_OK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
ASSERT_GE(kCFCount, 5);
// merge operator
{
std::shared_ptr<MergeOperator> merge_op =
cf_descs[0].options.merge_operator;
ASSERT_NE(merge_op.get(), nullptr);
cf_descs[0].options.merge_operator.reset();
ASSERT_NOK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
cf_descs[0].options.merge_operator.reset(new DummyMergeOperator());
ASSERT_NOK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
cf_descs[0].options.merge_operator = merge_op;
ASSERT_OK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
}
// prefix extractor
{
std::shared_ptr<const SliceTransform> prefix_extractor =
cf_descs[1].options.prefix_extractor;
ASSERT_NE(prefix_extractor, nullptr);
cf_descs[1].options.prefix_extractor.reset();
ASSERT_NOK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
cf_descs[1].options.prefix_extractor.reset(new DummySliceTransform());
ASSERT_NOK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
cf_descs[1].options.prefix_extractor = prefix_extractor;
ASSERT_OK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
}
// comparator
{
test::SimpleSuffixReverseComparator comparator;
auto* prev_comparator = cf_descs[2].options.comparator;
cf_descs[2].options.comparator = &comparator;
ASSERT_NOK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
cf_descs[2].options.comparator = prev_comparator;
ASSERT_OK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
}
// table factory
{
std::shared_ptr<TableFactory> table_factory =
cf_descs[3].options.table_factory;
ASSERT_NE(table_factory, nullptr);
cf_descs[3].options.table_factory.reset(new DummyTableFactory());
ASSERT_NOK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
cf_descs[3].options.table_factory = table_factory;
ASSERT_OK(
CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
}
}
} // namespace rocksdb } // namespace rocksdb
int main(int argc, char** argv) { int main(int argc, char** argv) {

Loading…
Cancel
Save