Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
main
Yueh-Hsuan Chiang
9 years ago
parent
e78389b554
commit
e11f676e34
@ -0,0 +1,113 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// This file demonstrates how to use the utility functions defined in
|
||||
// rocksdb/utilities/options_util.h to open a rocksdb database without
|
||||
// remembering all the rocksdb options.
|
||||
#include <cstdio> |
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
#include "rocksdb/cache.h" |
||||
#include "rocksdb/compaction_filter.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/slice.h" |
||||
#include "rocksdb/table.h" |
||||
#include "rocksdb/utilities/options_util.h" |
||||
|
||||
using namespace rocksdb; |
||||
|
||||
std::string kDBPath = "/tmp/rocksdb_options_file_example"; |
||||
|
||||
namespace { |
||||
// A dummy compaction filter
|
||||
class DummyCompactionFilter : public CompactionFilter { |
||||
public: |
||||
virtual ~DummyCompactionFilter() {} |
||||
virtual bool Filter(int level, const Slice& key, const Slice& existing_value, |
||||
std::string* new_value, bool* value_changed) const { |
||||
return false; |
||||
} |
||||
virtual const char* Name() const { return "DummyCompactionFilter"; } |
||||
}; |
||||
|
||||
} // namespace
|
||||
|
||||
int main() { |
||||
DBOptions db_opt; |
||||
db_opt.create_if_missing = true; |
||||
|
||||
std::vector<ColumnFamilyDescriptor> cf_descs; |
||||
cf_descs.push_back({kDefaultColumnFamilyName, ColumnFamilyOptions()}); |
||||
cf_descs.push_back({"new_cf", ColumnFamilyOptions()}); |
||||
|
||||
// initialize BlockBasedTableOptions
|
||||
auto cache = NewLRUCache(1 * 1024 * 1024 * 1024); |
||||
BlockBasedTableOptions bbt_opts; |
||||
bbt_opts.block_size = 32 * 1024; |
||||
bbt_opts.block_cache = cache; |
||||
|
||||
// initialize column families options
|
||||
std::unique_ptr<CompactionFilter> compaction_filter; |
||||
compaction_filter.reset(new DummyCompactionFilter()); |
||||
cf_descs[0].options.table_factory.reset(NewBlockBasedTableFactory(bbt_opts)); |
||||
cf_descs[0].options.compaction_filter = compaction_filter.get(); |
||||
cf_descs[1].options.table_factory.reset(NewBlockBasedTableFactory(bbt_opts)); |
||||
|
||||
// destroy and open DB
|
||||
DB* db; |
||||
Status s = DestroyDB(kDBPath, Options(db_opt, cf_descs[0].options)); |
||||
assert(s.ok()); |
||||
s = DB::Open(Options(db_opt, cf_descs[0].options), kDBPath, &db); |
||||
assert(s.ok()); |
||||
|
||||
// Create column family, and rocksdb will persist the options.
|
||||
ColumnFamilyHandle* cf; |
||||
s = db->CreateColumnFamily(ColumnFamilyOptions(), "new_cf", &cf); |
||||
assert(s.ok()); |
||||
|
||||
// close DB
|
||||
delete cf; |
||||
delete db; |
||||
|
||||
// In the following code, we will reopen the rocksdb instance using
|
||||
// the options file stored in the db directory.
|
||||
|
||||
// Load the options file.
|
||||
DBOptions loaded_db_opt; |
||||
std::vector<ColumnFamilyDescriptor> loaded_cf_descs; |
||||
s = LoadLatestOptions(kDBPath, Env::Default(), &loaded_db_opt, |
||||
&loaded_cf_descs); |
||||
assert(s.ok()); |
||||
assert(loaded_db_opt.create_if_missing == db_opt.create_if_missing); |
||||
|
||||
// Initialize pointer options for each column family
|
||||
for (size_t i = 0; i < loaded_cf_descs.size(); ++i) { |
||||
auto* loaded_bbt_opt = reinterpret_cast<BlockBasedTableOptions*>( |
||||
loaded_cf_descs[0].options.table_factory->GetOptions()); |
||||
// Expect the same as BlockBasedTableOptions will be loaded form file.
|
||||
assert(loaded_bbt_opt->block_size == bbt_opts.block_size); |
||||
// However, block_cache needs to be manually initialized as documented
|
||||
// in rocksdb/utilities/options_util.h.
|
||||
loaded_bbt_opt->block_cache = cache; |
||||
} |
||||
// In addition, as pointer options are initialized with default value,
|
||||
// we need to properly initialized all the pointer options if non-defalut
|
||||
// values are used before calling DB::Open().
|
||||
assert(loaded_cf_descs[0].options.compaction_filter == nullptr); |
||||
loaded_cf_descs[0].options.compaction_filter = compaction_filter.get(); |
||||
|
||||
// reopen the db using the loaded options.
|
||||
std::vector<ColumnFamilyHandle*> handles; |
||||
s = DB::Open(loaded_db_opt, kDBPath, loaded_cf_descs, &handles, &db); |
||||
assert(s.ok()); |
||||
|
||||
// close DB
|
||||
for (auto* handle : handles) { |
||||
delete handle; |
||||
} |
||||
delete db; |
||||
} |
||||
@ -0,0 +1,65 @@ |
||||
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
// This file contains utility functions for RocksDB Options.
|
||||
#pragma once |
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
|
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/status.h" |
||||
|
||||
namespace rocksdb { |
||||
// Constructs the DBOptions and ColumnFamilyDescriptors by loading the
|
||||
// latest RocksDB options file stored in the specified rocksdb database.
|
||||
//
|
||||
// Note that the all the pointer options (except table_factory, which will
|
||||
// be described in more details below) will be initialized with the default
|
||||
// values. Developers can further initialize them after this function call.
|
||||
// Below is an example list of pointer options which will be initialized
|
||||
//
|
||||
// * env
|
||||
// * memtable_factory
|
||||
// * compaction_filter_factory
|
||||
// * prefix_extractor
|
||||
// * comparator
|
||||
// * merge_operator
|
||||
// * compaction_filter
|
||||
//
|
||||
// For table_factory, this function further supports deserializing
|
||||
// BlockBasedTableFactory and its BlockBasedTableOptions except the
|
||||
// pointer options of BlockBasedTableOptions (flush_block_policy_factory,
|
||||
// block_cache, and block_cache_compressed), which will be initialized with
|
||||
// default values. Developers can further specify these three options by
|
||||
// casting the return value of TableFactoroy::GetOptions() to
|
||||
// BlockBasedTableOptions and making necessary changes.
|
||||
//
|
||||
// examples/options_file_example.cc demonstrates how to use this function
|
||||
// to open a RocksDB instance.
|
||||
//
|
||||
// @see LoadOptionsFromFile
|
||||
Status LoadLatestOptions(const std::string& dbpath, Env* env, |
||||
DBOptions* db_options, |
||||
std::vector<ColumnFamilyDescriptor>* cf_descs); |
||||
|
||||
// Similar to LoadLatestOptions, this function constructs the DBOptions
|
||||
// and ColumnFamilyDescriptors based on the specified RocksDB Options file.
|
||||
//
|
||||
// @see LoadLatestOptions
|
||||
Status LoadOptionsFromFile(const std::string& options_file_name, Env* env, |
||||
DBOptions* db_options, |
||||
std::vector<ColumnFamilyDescriptor>* cf_descs); |
||||
|
||||
// Returns the latest options file name under the specified db path.
|
||||
Status GetLatestOptionsFileName(const std::string& dbpath, Env* env, |
||||
std::string* options_file_name); |
||||
|
||||
} // namespace rocksdb
|
||||
#endif // !ROCKSDB_LITE
|
||||
@ -0,0 +1,76 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
|
||||
#include "rocksdb/utilities/options_util.h" |
||||
|
||||
#include "db/filename.h" |
||||
#include "rocksdb/options.h" |
||||
#include "util/options_parser.h" |
||||
|
||||
namespace rocksdb { |
||||
Status LoadOptionsFromFile(const std::string& file_name, Env* env, |
||||
DBOptions* db_options, |
||||
std::vector<ColumnFamilyDescriptor>* cf_descs) { |
||||
RocksDBOptionsParser parser; |
||||
Status s = parser.Parse(file_name, env); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
*db_options = *parser.db_opt(); |
||||
|
||||
const std::vector<std::string>& cf_names = *parser.cf_names(); |
||||
const std::vector<ColumnFamilyOptions>& cf_opts = *parser.cf_opts(); |
||||
cf_descs->clear(); |
||||
for (size_t i = 0; i < cf_opts.size(); ++i) { |
||||
cf_descs->push_back({cf_names[i], cf_opts[i]}); |
||||
} |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status GetLatestOptionsFileName(const std::string& dbpath, |
||||
Env* env, std::string* options_file_name) { |
||||
Status s; |
||||
std::string latest_file_name; |
||||
uint64_t latest_time_stamp = 0; |
||||
std::vector<std::string> file_names; |
||||
s = env->GetChildren(dbpath, &file_names); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
for (auto& file_name : file_names) { |
||||
uint64_t time_stamp; |
||||
FileType type; |
||||
if (ParseFileName(file_name, &time_stamp, &type) && type == kOptionsFile) { |
||||
if (time_stamp > latest_time_stamp) { |
||||
latest_time_stamp = time_stamp; |
||||
latest_file_name = file_name; |
||||
} |
||||
} |
||||
} |
||||
if (latest_file_name.size() == 0) { |
||||
return Status::NotFound("No options files found in the DB directory."); |
||||
} |
||||
*options_file_name = latest_file_name; |
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status LoadLatestOptions(const std::string& dbpath, Env* env, |
||||
DBOptions* db_options, |
||||
std::vector<ColumnFamilyDescriptor>* cf_descs) { |
||||
std::string options_file_name; |
||||
Status s = GetLatestOptionsFileName(dbpath, env, &options_file_name); |
||||
if (!s.ok()) { |
||||
return s; |
||||
} |
||||
|
||||
return LoadOptionsFromFile(dbpath + "/" + options_file_name, env, |
||||
db_options, cf_descs); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
#endif // !ROCKSDB_LITE
|
||||
@ -0,0 +1,102 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
#ifndef __STDC_FORMAT_MACROS |
||||
#define __STDC_FORMAT_MACROS |
||||
#endif |
||||
|
||||
#include <inttypes.h> |
||||
|
||||
#include <cctype> |
||||
#include <unordered_map> |
||||
|
||||
#include "rocksdb/utilities/options_util.h" |
||||
#include "util/options_parser.h" |
||||
#include "util/random.h" |
||||
#include "util/testharness.h" |
||||
#include "util/testutil.h" |
||||
|
||||
#ifndef GFLAGS |
||||
bool FLAGS_enable_print = false; |
||||
#else |
||||
#include <gflags/gflags.h> |
||||
using GFLAGS::ParseCommandLineFlags; |
||||
DEFINE_bool(enable_print, false, "Print options generated to console."); |
||||
#endif // GFLAGS
|
||||
|
||||
namespace rocksdb { |
||||
class OptionsUtilTest : public testing::Test { |
||||
public: |
||||
OptionsUtilTest() { env_.reset(new test::StringEnv(Env::Default())); } |
||||
|
||||
protected: |
||||
std::unique_ptr<test::StringEnv> env_; |
||||
}; |
||||
|
||||
bool IsBlockBasedTableFactory(TableFactory* tf) { |
||||
return tf->Name() == BlockBasedTableFactory().Name(); |
||||
} |
||||
|
||||
TEST_F(OptionsUtilTest, SaveAndLoad) { |
||||
const size_t kCFCount = 5; |
||||
Random rnd(0xFB); |
||||
|
||||
DBOptions db_opt; |
||||
std::vector<std::string> cf_names; |
||||
std::vector<ColumnFamilyOptions> cf_opts; |
||||
test::RandomInitDBOptions(&db_opt, &rnd); |
||||
for (size_t i = 0; i < kCFCount; ++i) { |
||||
cf_names.push_back(i == 0 ? kDefaultColumnFamilyName |
||||
: test::RandomName(&rnd, 10)); |
||||
cf_opts.emplace_back(); |
||||
test::RandomInitCFOptions(&cf_opts.back(), &rnd); |
||||
} |
||||
|
||||
const std::string kFileName = "OPTIONS-123456"; |
||||
PersistRocksDBOptions(db_opt, cf_names, cf_opts, kFileName, env_.get()); |
||||
|
||||
DBOptions loaded_db_opt; |
||||
std::vector<ColumnFamilyDescriptor> loaded_cf_descs; |
||||
ASSERT_OK(LoadOptionsFromFile(kFileName, env_.get(), &loaded_db_opt, |
||||
&loaded_cf_descs)); |
||||
|
||||
ASSERT_OK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); |
||||
test::RandomInitDBOptions(&db_opt, &rnd); |
||||
ASSERT_NOK(RocksDBOptionsParser::VerifyDBOptions(db_opt, loaded_db_opt)); |
||||
|
||||
for (size_t i = 0; i < kCFCount; ++i) { |
||||
ASSERT_EQ(cf_names[i], loaded_cf_descs[i].name); |
||||
ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions( |
||||
cf_opts[i], loaded_cf_descs[i].options)); |
||||
if (IsBlockBasedTableFactory(cf_opts[i].table_factory.get())) { |
||||
ASSERT_OK(RocksDBOptionsParser::VerifyTableFactory( |
||||
cf_opts[i].table_factory.get(), |
||||
loaded_cf_descs[i].options.table_factory.get())); |
||||
} |
||||
test::RandomInitCFOptions(&cf_opts[i], &rnd); |
||||
ASSERT_NOK(RocksDBOptionsParser::VerifyCFOptions( |
||||
cf_opts[i], loaded_cf_descs[i].options)); |
||||
} |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
#ifdef GFLAGS |
||||
ParseCommandLineFlags(&argc, &argv, true); |
||||
#endif // GFLAGS
|
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
|
||||
#else |
||||
#include <cstdio> |
||||
|
||||
int main(int argc, char** argv) { |
||||
printf("Skipped in RocksDBLite as utilities are not supported.\n"); |
||||
return 0; |
||||
} |
||||
#endif // !ROCKSDB_LITE
|
||||
Loading…
Reference in new issue