Add CheckOptionsCompatibility() API to options_util

Summary:
Add CheckOptionsCompatibility() API to options_util that returns
Status::OK if the input DBOptions and ColumnFamilyDescriptors
are compatible with the latest options stored in the specified DB path.

Test Plan: Added tests in options_util_test

Reviewers: igor, anthony, IslamAbdelRahman, rven, sdong

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D50649

HISTORY.md

@@ -7,6 +7,7 @@
 * Add MemoryUtil in rocksdb/utilities/memory.h.  It currently offers a way to get the memory usage by type from a list rocksdb instances.
 * RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions.
 * Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h.  This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance.
+* Introduce CheckOptionsCompatibility() in rocksdb/utilities/options_util.h.  This function checks whether the input set of options is able to open the specified DB successfully.
 ### Public API Changes
 * CompactionFilter::Context includes information of Column Family ID
 * The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower.

include/rocksdb/utilities/options_util.h

@@ -61,5 +61,20 @@ Status LoadOptionsFromFile(const std::string& options_file_name, Env* env,
 Status GetLatestOptionsFileName(const std::string& dbpath, Env* env,
                                 std::string* options_file_name);
 
+// Returns Status::OK if the input DBOptions and ColumnFamilyDescriptors
+// are compatible with the latest options stored in the specified DB path.
+//
+// If the return status is non-ok, it means the specified RocksDB instance
+// might not be correctly opened with the input set of options.  Currently,
+// changing one of the following options will fail the compatibility check:
+//
+// * comparator
+// * prefix_extractor
+// * table_factory
+// * merge_operator
+Status CheckOptionsCompatibility(
+    const std::string& dbpath, Env* env, const DBOptions& db_options,
+    const std::vector<ColumnFamilyDescriptor>& cf_descs);
+
 }  // namespace rocksdb
 #endif  // !ROCKSDB_LITE

utilities/options/options_util.cc

@@ -72,5 +72,28 @@ Status LoadLatestOptions(const std::string& dbpath, Env* env,
                              db_options, cf_descs);
 }
 
+Status CheckOptionsCompatibility(
+    const std::string& dbpath, Env* env, const DBOptions& db_options,
+    const std::vector<ColumnFamilyDescriptor>& cf_descs) {
+  std::string options_file_name;
+  Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name);
+  if (!s.ok()) {
+    return s;
+  }
+
+  std::vector<std::string> cf_names;
+  std::vector<ColumnFamilyOptions> cf_opts;
+  for (const auto& cf_desc : cf_descs) {
+    cf_names.push_back(cf_desc.name);
+    cf_opts.push_back(cf_desc.options);
+  }
+
+  const OptionsSanityCheckLevel kDefaultLevel = kSanityLevelLooselyCompatible;
+
+  return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
+      db_options, cf_names, cf_opts, dbpath + "/" + options_file_name, env,
+      kDefaultLevel);
+}
+
 }  // namespace rocksdb
 #endif  // !ROCKSDB_LITE

utilities/options/options_util_test.cc

@@ -13,6 +13,8 @@
 #include <cctype>
 #include <unordered_map>
 
+#include "rocksdb/db.h"
+#include "rocksdb/table.h"
 #include "rocksdb/utilities/options_util.h"
 #include "util/options_parser.h"
 #include "util/random.h"
@@ -30,10 +32,15 @@ DEFINE_bool(enable_print, false, "Print options generated to console.");
 namespace rocksdb {
 class OptionsUtilTest : public testing::Test {
  public:
-  OptionsUtilTest() { env_.reset(new test::StringEnv(Env::Default())); }
+  OptionsUtilTest() : rnd_(0xFB) {
+    env_.reset(new test::StringEnv(Env::Default()));
+    dbname_ = test::TmpDir() + "/options_util_test";
+  }
 
  protected:
   std::unique_ptr<test::StringEnv> env_;
+  std::string dbname_;
+  Random rnd_;
 };
 
 bool IsBlockBasedTableFactory(TableFactory* tf) {
@@ -42,17 +49,16 @@ bool IsBlockBasedTableFactory(TableFactory* tf) {
 
 TEST_F(OptionsUtilTest, SaveAndLoad) {
   const size_t kCFCount = 5;
-  Random rnd(0xFB);
 
   DBOptions db_opt;
   std::vector<std::string> cf_names;
   std::vector<ColumnFamilyOptions> cf_opts;
-  test::RandomInitDBOptions(&db_opt, &rnd);
+  test::RandomInitDBOptions(&db_opt, &rnd_);
   for (size_t i = 0; i < kCFCount; ++i) {
     cf_names.push_back(i == 0 ? kDefaultColumnFamilyName
-                              : test::RandomName(&rnd, 10));
+                              : test::RandomName(&rnd_, 10));
     cf_opts.emplace_back();
-    test::RandomInitCFOptions(&cf_opts.back(), &rnd);
+    test::RandomInitCFOptions(&cf_opts.back(), &rnd_);
   }
 
   const std::string kFileName = "OPTIONS-123456";
@@ -64,7 +70,7 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
                                 &loaded_cf_descs));
 
   ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt));
-  test::RandomInitDBOptions(&db_opt, &rnd);
+  test::RandomInitDBOptions(&db_opt, &rnd_);
   ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt));
 
   for (size_t i = 0; i < kCFCount; ++i) {
@@ -76,7 +82,7 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
           cf_opts[i].table_factory.get(),
           loaded_cf_descs[i].options.table_factory.get()));
     }
-    test::RandomInitCFOptions(&cf_opts[i], &rnd);
+    test::RandomInitCFOptions(&cf_opts[i], &rnd_);
     ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions(
         cf_opts[i], loaded_cf_descs[i].options));
   }
@@ -88,6 +94,179 @@ TEST_F(OptionsUtilTest, SaveAndLoad) {
   }
 }
 
+namespace {
+class DummyTableFactory : public TableFactory {
+ public:
+  DummyTableFactory() {}
+  virtual ~DummyTableFactory() {}
+
+  virtual const char* Name() const { return "DummyTableFactory"; }
+
+  virtual Status NewTableReader(const TableReaderOptions& table_reader_options,
+                                unique_ptr<RandomAccessFileReader>&& file,
+                                uint64_t file_size,
+                                unique_ptr<TableReader>* table_reader) const {
+    return Status::NotSupported();
+  }
+
+  virtual TableBuilder* NewTableBuilder(
+      const TableBuilderOptions& table_builder_options,
+      uint32_t column_family_id, WritableFileWriter* file) const {
+    return nullptr;
+  }
+
+  virtual Status SanitizeOptions(const DBOptions& db_opts,
+                                 const ColumnFamilyOptions& cf_opts) const {
+    return Status::NotSupported();
+  }
+
+  virtual std::string GetPrintableTableOptions() const { return ""; }
+};
+
+class DummyMergeOperator : public MergeOperator {
+ public:
+  DummyMergeOperator() {}
+  virtual ~DummyMergeOperator() {}
+
+  virtual bool FullMerge(const Slice& key, const Slice* existing_value,
+                         const std::deque<std::string>& operand_list,
+                         std::string* new_value, Logger* logger) const {
+    return false;
+  }
+
+  virtual bool PartialMergeMulti(const Slice& key,
+                                 const std::deque<Slice>& operand_list,
+                                 std::string* new_value, Logger* logger) const {
+    return false;
+  }
+
+  virtual const char* Name() const { return "DummyMergeOperator"; }
+};
+
+class DummySliceTransform : public SliceTransform {
+ public:
+  DummySliceTransform() {}
+  virtual ~DummySliceTransform() {}
+
+  // Return the name of this transformation.
+  virtual const char* Name() const { return "DummySliceTransform"; }
+
+  // transform a src in domain to a dst in the range
+  virtual Slice Transform(const Slice& src) const { return src; }
+
+  // determine whether this is a valid src upon the function applies
+  virtual bool InDomain(const Slice& src) const { return false; }
+
+  // determine whether dst=Transform(src) for some src
+  virtual bool InRange(const Slice& dst) const { return false; }
+};
+
+}  // namespace
+
+TEST_F(OptionsUtilTest, SanityCheck) {
+  DBOptions db_opt;
+  std::vector<ColumnFamilyDescriptor> cf_descs;
+  const size_t kCFCount = 5;
+  for (size_t i = 0; i < kCFCount; ++i) {
+    cf_descs.emplace_back();
+    cf_descs.back().name =
+        (i == 0) ? kDefaultColumnFamilyName : test::RandomName(&rnd_, 10);
+
+    cf_descs.back().options.table_factory.reset(NewBlockBasedTableFactory());
+    cf_descs.back().options.prefix_extractor.reset(
+        test::RandomSliceTransform(&rnd_));
+    cf_descs.back().options.merge_operator.reset(
+        test::RandomMergeOperator(&rnd_));
+  }
+
+  db_opt.create_missing_column_families = true;
+  db_opt.create_if_missing = true;
+
+  DestroyDB(dbname_, Options(db_opt, cf_descs[0].options));
+  DB* db;
+  std::vector<ColumnFamilyHandle*> handles;
+  // open and persist the options
+  ASSERT_OK(DB::Open(db_opt, dbname_, cf_descs, &handles, &db));
+
+  // close the db
+  for (auto* handle : handles) {
+    delete handle;
+  }
+  delete db;
+
+  // perform sanity check
+  ASSERT_OK(
+      CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+
+  ASSERT_GE(kCFCount, 5);
+  // merge operator
+  {
+    std::shared_ptr<MergeOperator> merge_op =
+        cf_descs[0].options.merge_operator;
+
+    ASSERT_NE(merge_op.get(), nullptr);
+    cf_descs[0].options.merge_operator.reset();
+    ASSERT_NOK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+
+    cf_descs[0].options.merge_operator.reset(new DummyMergeOperator());
+    ASSERT_NOK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+
+    cf_descs[0].options.merge_operator = merge_op;
+    ASSERT_OK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+  }
+
+  // prefix extractor
+  {
+    std::shared_ptr<const SliceTransform> prefix_extractor =
+        cf_descs[1].options.prefix_extractor;
+
+    ASSERT_NE(prefix_extractor, nullptr);
+    cf_descs[1].options.prefix_extractor.reset();
+    ASSERT_NOK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+
+    cf_descs[1].options.prefix_extractor.reset(new DummySliceTransform());
+    ASSERT_NOK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+
+    cf_descs[1].options.prefix_extractor = prefix_extractor;
+    ASSERT_OK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+  }
+
+  // comparator
+  {
+    test::SimpleSuffixReverseComparator comparator;
+
+    auto* prev_comparator = cf_descs[2].options.comparator;
+    cf_descs[2].options.comparator = &comparator;
+    ASSERT_NOK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+
+    cf_descs[2].options.comparator = prev_comparator;
+    ASSERT_OK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+  }
+
+  // table factory
+  {
+    std::shared_ptr<TableFactory> table_factory =
+        cf_descs[3].options.table_factory;
+
+    ASSERT_NE(table_factory, nullptr);
+    cf_descs[3].options.table_factory.reset(new DummyTableFactory());
+    ASSERT_NOK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+
+    cf_descs[3].options.table_factory = table_factory;
+    ASSERT_OK(
+        CheckOptionsCompatibility(dbname_, Env::Default(), db_opt, cf_descs));
+  }
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {