Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
main
Yueh-Hsuan Chiang
9 years ago
parent
e78389b554
commit
e11f676e34
@ -0,0 +1,113 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// This file demonstrates how to use the utility functions defined in
|
||||||
|
// rocksdb/utilities/options_util.h to open a rocksdb database without
|
||||||
|
// remembering all the rocksdb options.
|
||||||
|
#include <cstdio> |
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "rocksdb/cache.h" |
||||||
|
#include "rocksdb/compaction_filter.h" |
||||||
|
#include "rocksdb/db.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/table.h" |
||||||
|
#include "rocksdb/utilities/options_util.h" |
||||||
|
|
||||||
|
using namespace rocksdb; |
||||||
|
|
||||||
|
std::string kDBPath = "/tmp/rocksdb_options_file_example"; |
||||||
|
|
||||||
|
namespace { |
||||||
|
// A dummy compaction filter
|
||||||
|
class DummyCompactionFilter : public CompactionFilter { |
||||||
|
public: |
||||||
|
virtual ~DummyCompactionFilter() {} |
||||||
|
virtual bool Filter(int level, const Slice& key, const Slice& existing_value, |
||||||
|
std::string* new_value, bool* value_changed) const { |
||||||
|
return false; |
||||||
|
} |
||||||
|
virtual const char* Name() const { return "DummyCompactionFilter"; } |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
int main() { |
||||||
|
DBOptions db_opt; |
||||||
|
db_opt.create_if_missing = true; |
||||||
|
|
||||||
|
std::vector<ColumnFamilyDescriptor> cf_descs; |
||||||
|
cf_descs.push_back({kDefaultColumnFamilyName, ColumnFamilyOptions()}); |
||||||
|
cf_descs.push_back({"new_cf", ColumnFamilyOptions()}); |
||||||
|
|
||||||
|
// initialize BlockBasedTableOptions
|
||||||
|
auto cache = NewLRUCache(1 * 1024 * 1024 * 1024); |
||||||
|
BlockBasedTableOptions bbt_opts; |
||||||
|
bbt_opts.block_size = 32 * 1024; |
||||||
|
bbt_opts.block_cache = cache; |
||||||
|
|
||||||
|
// initialize column families options
|
||||||
|
std::unique_ptr<CompactionFilter> compaction_filter; |
||||||
|
compaction_filter.reset(new DummyCompactionFilter()); |
||||||
|
cf_descs[0].options.table_factory.reset(NewBlockBasedTableFactory(bbt_opts)); |
||||||
|
cf_descs[0].options.compaction_filter = compaction_filter.get(); |
||||||
|
cf_descs[1].options.table_factory.reset(NewBlockBasedTableFactory(bbt_opts)); |
||||||
|
|
||||||
|
// destroy and open DB
|
||||||
|
DB* db; |
||||||
|
Status s = DestroyDB(kDBPath, Options(db_opt, cf_descs[0].options)); |
||||||
|
assert(s.ok()); |
||||||
|
s = DB::Open(Options(db_opt, cf_descs[0].options), kDBPath, &db); |
||||||
|
assert(s.ok()); |
||||||
|
|
||||||
|
// Create column family, and rocksdb will persist the options.
|
||||||
|
ColumnFamilyHandle* cf; |
||||||
|
s = db->CreateColumnFamily(ColumnFamilyOptions(), "new_cf", &cf); |
||||||
|
assert(s.ok()); |
||||||
|
|
||||||
|
// close DB
|
||||||
|
delete cf; |
||||||
|
delete db; |
||||||
|
|
||||||
|
// In the following code, we will reopen the rocksdb instance using
|
||||||
|
// the options file stored in the db directory.
|
||||||
|
|
||||||
|
// Load the options file.
|
||||||
|
DBOptions loaded_db_opt; |
||||||
|
std::vector<ColumnFamilyDescriptor> loaded_cf_descs; |
||||||
|
s = LoadLatestOptions(kDBPath, Env::Default(), &loaded_db_opt, |
||||||
|
&loaded_cf_descs); |
||||||
|
assert(s.ok()); |
||||||
|
assert(loaded_db_opt.create_if_missing == db_opt.create_if_missing); |
||||||
|
|
||||||
|
// Initialize pointer options for each column family
|
||||||
|
for (size_t i = 0; i < loaded_cf_descs.size(); ++i) { |
||||||
|
auto* loaded_bbt_opt = reinterpret_cast<BlockBasedTableOptions*>( |
||||||
|
loaded_cf_descs[0].options.table_factory->GetOptions()); |
||||||
|
// Expect the same as BlockBasedTableOptions will be loaded form file.
|
||||||
|
assert(loaded_bbt_opt->block_size == bbt_opts.block_size); |
||||||
|
// However, block_cache needs to be manually initialized as documented
|
||||||
|
// in rocksdb/utilities/options_util.h.
|
||||||
|
loaded_bbt_opt->block_cache = cache; |
||||||
|
} |
||||||
|
// In addition, as pointer options are initialized with default value,
|
||||||
|
// we need to properly initialized all the pointer options if non-defalut
|
||||||
|
// values are used before calling DB::Open().
|
||||||
|
assert(loaded_cf_descs[0].options.compaction_filter == nullptr); |
||||||
|
loaded_cf_descs[0].options.compaction_filter = compaction_filter.get(); |
||||||
|
|
||||||
|
// reopen the db using the loaded options.
|
||||||
|
std::vector<ColumnFamilyHandle*> handles; |
||||||
|
s = DB::Open(loaded_db_opt, kDBPath, loaded_cf_descs, &handles, &db); |
||||||
|
assert(s.ok()); |
||||||
|
|
||||||
|
// close DB
|
||||||
|
for (auto* handle : handles) { |
||||||
|
delete handle; |
||||||
|
} |
||||||
|
delete db; |
||||||
|
} |
||||||
@ -0,0 +1,65 @@ |
|||||||
|
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
// This file contains utility functions for RocksDB Options.
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "rocksdb/db.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
// Constructs the DBOptions and ColumnFamilyDescriptors by loading the
|
||||||
|
// latest RocksDB options file stored in the specified rocksdb database.
|
||||||
|
//
|
||||||
|
// Note that the all the pointer options (except table_factory, which will
|
||||||
|
// be described in more details below) will be initialized with the default
|
||||||
|
// values. Developers can further initialize them after this function call.
|
||||||
|
// Below is an example list of pointer options which will be initialized
|
||||||
|
//
|
||||||
|
// * env
|
||||||
|
// * memtable_factory
|
||||||
|
// * compaction_filter_factory
|
||||||
|
// * prefix_extractor
|
||||||
|
// * comparator
|
||||||
|
// * merge_operator
|
||||||
|
// * compaction_filter
|
||||||
|
//
|
||||||
|
// For table_factory, this function further supports deserializing
|
||||||
|
// BlockBasedTableFactory and its BlockBasedTableOptions except the
|
||||||
|
// pointer options of BlockBasedTableOptions (flush_block_policy_factory,
|
||||||
|
// block_cache, and block_cache_compressed), which will be initialized with
|
||||||
|
// default values. Developers can further specify these three options by
|
||||||
|
// casting the return value of TableFactoroy::GetOptions() to
|
||||||
|
// BlockBasedTableOptions and making necessary changes.
|
||||||
|
//
|
||||||
|
// examples/options_file_example.cc demonstrates how to use this function
|
||||||
|
// to open a RocksDB instance.
|
||||||
|
//
|
||||||
|
// @see LoadOptionsFromFile
|
||||||
|
Status LoadLatestOptions(const std::string& dbpath, Env* env, |
||||||
|
DBOptions* db_options, |
||||||
|
std::vector<ColumnFamilyDescriptor>* cf_descs); |
||||||
|
|
||||||
|
// Similar to LoadLatestOptions, this function constructs the DBOptions
|
||||||
|
// and ColumnFamilyDescriptors based on the specified RocksDB Options file.
|
||||||
|
//
|
||||||
|
// @see LoadLatestOptions
|
||||||
|
Status LoadOptionsFromFile(const std::string& options_file_name, Env* env, |
||||||
|
DBOptions* db_options, |
||||||
|
std::vector<ColumnFamilyDescriptor>* cf_descs); |
||||||
|
|
||||||
|
// Returns the latest options file name under the specified db path.
|
||||||
|
Status GetLatestOptionsFileName(const std::string& dbpath, Env* env, |
||||||
|
std::string* options_file_name); |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // !ROCKSDB_LITE
|
||||||
@ -0,0 +1,76 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "rocksdb/utilities/options_util.h" |
||||||
|
|
||||||
|
#include "db/filename.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "util/options_parser.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
Status LoadOptionsFromFile(const std::string& file_name, Env* env, |
||||||
|
DBOptions* db_options, |
||||||
|
std::vector<ColumnFamilyDescriptor>* cf_descs) { |
||||||
|
RocksDBOptionsParser parser; |
||||||
|
Status s = parser.Parse(file_name, env); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
*db_options = *parser.db_opt(); |
||||||
|
|
||||||
|
const std::vector<std::string>& cf_names = *parser.cf_names(); |
||||||
|
const std::vector<ColumnFamilyOptions>& cf_opts = *parser.cf_opts(); |
||||||
|
cf_descs->clear(); |
||||||
|
for (size_t i = 0; i < cf_opts.size(); ++i) { |
||||||
|
cf_descs->push_back({cf_names[i], cf_opts[i]}); |
||||||
|
} |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status GetLatestOptionsFileName(const std::string& dbpath, |
||||||
|
Env* env, std::string* options_file_name) { |
||||||
|
Status s; |
||||||
|
std::string latest_file_name; |
||||||
|
uint64_t latest_time_stamp = 0; |
||||||
|
std::vector<std::string> file_names; |
||||||
|
s = env->GetChildren(dbpath, &file_names); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
for (auto& file_name : file_names) { |
||||||
|
uint64_t time_stamp; |
||||||
|
FileType type; |
||||||
|
if (ParseFileName(file_name, &time_stamp, &type) && type == kOptionsFile) { |
||||||
|
if (time_stamp > latest_time_stamp) { |
||||||
|
latest_time_stamp = time_stamp; |
||||||
|
latest_file_name = file_name; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
if (latest_file_name.size() == 0) { |
||||||
|
return Status::NotFound("No options files found in the DB directory."); |
||||||
|
} |
||||||
|
*options_file_name = latest_file_name; |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status LoadLatestOptions(const std::string& dbpath, Env* env, |
||||||
|
DBOptions* db_options, |
||||||
|
std::vector<ColumnFamilyDescriptor>* cf_descs) { |
||||||
|
std::string options_file_name; |
||||||
|
Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name); |
||||||
|
if (!s.ok()) { |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
return LoadOptionsFromFile(dbpath + "/" + options_file_name, env, |
||||||
|
db_options, cf_descs); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // !ROCKSDB_LITE
|
||||||
@ -0,0 +1,102 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
#ifndef __STDC_FORMAT_MACROS |
||||||
|
#define __STDC_FORMAT_MACROS |
||||||
|
#endif |
||||||
|
|
||||||
|
#include <inttypes.h> |
||||||
|
|
||||||
|
#include <cctype> |
||||||
|
#include <unordered_map> |
||||||
|
|
||||||
|
#include "rocksdb/utilities/options_util.h" |
||||||
|
#include "util/options_parser.h" |
||||||
|
#include "util/random.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
#include "util/testutil.h" |
||||||
|
|
||||||
|
#ifndef GFLAGS |
||||||
|
bool FLAGS_enable_print = false; |
||||||
|
#else |
||||||
|
#include <gflags/gflags.h> |
||||||
|
using GFLAGS::ParseCommandLineFlags; |
||||||
|
DEFINE_bool(enable_print, false, "Print options generated to console."); |
||||||
|
#endif // GFLAGS
|
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
class OptionsUtilTest : public testing::Test { |
||||||
|
public: |
||||||
|
OptionsUtilTest() { env_.reset(new test::StringEnv(Env::Default())); } |
||||||
|
|
||||||
|
protected: |
||||||
|
std::unique_ptr<test::StringEnv> env_; |
||||||
|
}; |
||||||
|
|
||||||
|
bool IsBlockBasedTableFactory(TableFactory* tf) { |
||||||
|
return tf->Name() == BlockBasedTableFactory().Name(); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(OptionsUtilTest, SaveAndLoad) { |
||||||
|
const size_t kCFCount = 5; |
||||||
|
Random rnd(0xFB); |
||||||
|
|
||||||
|
DBOptions db_opt; |
||||||
|
std::vector<std::string> cf_names; |
||||||
|
std::vector<ColumnFamilyOptions> cf_opts; |
||||||
|
test::RandomInitDBOptions(&db_opt, &rnd); |
||||||
|
for (size_t i = 0; i < kCFCount; ++i) { |
||||||
|
cf_names.push_back(i == 0 ? kDefaultColumnFamilyName |
||||||
|
: test::RandomName(&rnd, 10)); |
||||||
|
cf_opts.emplace_back(); |
||||||
|
test::RandomInitCFOptions(&cf_opts.back(), &rnd); |
||||||
|
} |
||||||
|
|
||||||
|
const std::string kFileName = "OPTIONS-123456"; |
||||||
|
PersistRocksDBOptions(db_opt, cf_names, cf_opts, kFileName, env_.get()); |
||||||
|
|
||||||
|
DBOptions loaded_db_opt; |
||||||
|
std::vector<ColumnFamilyDescriptor> loaded_cf_descs; |
||||||
|
ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt, |
||||||
|
&loaded_cf_descs)); |
||||||
|
|
||||||
|
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); |
||||||
|
test::RandomInitDBOptions(&db_opt, &rnd); |
||||||
|
ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); |
||||||
|
|
||||||
|
for (size_t i = 0; i < kCFCount; ++i) { |
||||||
|
ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name); |
||||||
|
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( |
||||||
|
cf_opts[i], loaded_cf_descs[i].options)); |
||||||
|
if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { |
||||||
|
ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory( |
||||||
|
cf_opts[i].table_factory.get(), |
||||||
|
loaded_cf_descs[i].options.table_factory.get())); |
||||||
|
} |
||||||
|
test::RandomInitCFOptions(&cf_opts[i], &rnd); |
||||||
|
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( |
||||||
|
cf_opts[i], loaded_cf_descs[i].options)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
::testing::InitGoogleTest(&argc, argv); |
||||||
|
#ifdef GFLAGS |
||||||
|
ParseCommandLineFlags(&argc, &argv, true); |
||||||
|
#endif // GFLAGS
|
||||||
|
return RUN_ALL_TESTS(); |
||||||
|
} |
||||||
|
|
||||||
|
#else |
||||||
|
#include <cstdio> |
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
printf("Skipped in RocksDBLite as utilities are not supported.\n"); |
||||||
|
return 0; |
||||||
|
} |
||||||
|
#endif // !ROCKSDB_LITE
|
||||||
Loading…
Reference in new issue