NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

DB::GetOptions() reflect dynamic changed options

Browse Source 
Summary: DB::GetOptions() reflect dynamic changed options.

Test Plan: See the new unit test.

Reviewers: yhchiang, sdong, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D63903
main
Yi Wu 8 years ago
parent 215d12826d
commit 17f76fc564
13 changed files with 144 additions and 77 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 4
      HISTORY.md
  2. 4
      db/column_family.cc
  3. 6
      db/column_family.h
  4. 8
      db/db_impl.cc
  5. 5
      db/db_impl.h
  6. 50
      db/db_options_test.cc
  7. 3
      db/db_test.cc
  8. 2
      include/rocksdb/convenience.h
  9. 5
      include/rocksdb/db.h
  10. 3
      include/rocksdb/utilities/stackable_db.h
  11. 115
      util/options_helper.cc
  12. 13
      util/options_helper.h
  13. 3
      utilities/backupable/backupable_db_test.cc

4
HISTORY.md
Unescape View File

@ -1,4 +1,8 @@
# Rocksdb Change Log
## Unreleased
### Public API Change
* DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.
## 4.12.0 (9/12/2016)
### Public API Change
* CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads.

4
db/column_family.cc
Unescape View File

@ -482,6 +482,10 @@ void ColumnFamilyData::SetDropped() {
column_family_set_->RemoveColumnFamily(this);
}
ColumnFamilyOptions ColumnFamilyData::GetLatestCFOptions() const {
return BuildColumnFamilyOptions(options_, mutable_cf_options_);
}
const double kSlowdownRatio = 1.2;
namespace {

6
db/column_family.h
Unescape View File

@ -218,6 +218,12 @@ class ColumnFamilyData {
const MutableCFOptions* GetLatestMutableCFOptions() const {
return &mutable_cf_options_;
}
// REQUIRES: DB mutex held
// Build ColumnFamiliesOptions with immutable options and latest mutable
// options.
ColumnFamilyOptions GetLatestCFOptions() const;
#ifndef ROCKSDB_LITE
// REQUIRES: DB mutex held
Status SetOptions(

8
db/db_impl.cc
Unescape View File

@ -5110,9 +5110,10 @@ Env* DBImpl::GetEnv() const {
return env_;
}
const Options& DBImpl::GetOptions(ColumnFamilyHandle* column_family) const {
Options DBImpl::GetOptions(ColumnFamilyHandle* column_family) const {
InstrumentedMutexLock l(&mutex_);
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
return *cfh->cfd()->options();
return Options(db_options_, cfh->cfd()->GetLatestCFOptions());
}
const DBOptions& DBImpl::GetDBOptions() const { return db_options_; }
@ -5993,8 +5994,7 @@ Status DBImpl::WriteOptionsFile() {
continue;
}
cf_names.push_back(cfd->GetName());
cf_opts.push_back(BuildColumnFamilyOptions(
*cfd->options(), *cfd->GetLatestMutableCFOptions()));
cf_opts.push_back(cfd->GetLatestCFOptions());
}
// Unlock during expensive operations. New writes cannot get here

5
db/db_impl.h
Unescape View File

@ -165,8 +165,7 @@ class DBImpl : public DB {
virtual const std::string& GetName() const override;
virtual Env* GetEnv() const override;
using DB::GetOptions;
virtual const Options& GetOptions(
ColumnFamilyHandle* column_family) const override;
virtual Options GetOptions(ColumnFamilyHandle* column_family) const override;
using DB::GetDBOptions;
virtual const DBOptions& GetDBOptions() const override;
using DB::Flush;
@ -717,7 +716,7 @@ class DBImpl : public DB {
// same time.
InstrumentedMutex options_files_mutex_;
// State below is protected by mutex_
InstrumentedMutex mutex_;
mutable InstrumentedMutex mutex_;
std::atomic<bool> shutting_down_;
// This condition variable is signaled on these conditions:

50
db/db_options_test.cc
Unescape View File

@ -8,21 +8,71 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <limits>
#include <string>
#include <unordered_map>
#include "db/column_family.h"
#include "db/db_test_util.h"
#include "port/stack_trace.h"
#include "rocksdb/convenience.h"
#include "util/options_helper.h"
#include "util/random.h"
#include "util/sync_point.h"
#include "util/testutil.h"
namespace rocksdb {
class DBOptionsTest : public DBTestBase {
public:
DBOptionsTest() : DBTestBase("/db_options_test") {}
#ifndef ROCKSDB_LITE
std::unordered_map<std::string, std::string> GetMutableCFOptionsMap(
const ColumnFamilyOptions& options) {
std::string options_str;
GetStringFromColumnFamilyOptions(&options_str, options);
std::unordered_map<std::string, std::string> options_map;
StringToMap(options_str, &options_map);
std::unordered_map<std::string, std::string> mutable_map;
for (const auto opt : cf_options_type_info) {
if (opt.second.is_mutable &&
opt.second.verification != OptionVerificationType::kDeprecated) {
mutable_map[opt.first] = options_map[opt.first];
}
}
return mutable_map;
}
std::unordered_map<std::string, std::string> GetRandomizedMutableCFOptionsMap(
Random* rnd) {
Options options;
test::RandomInitCFOptions(&options, rnd);
auto sanitized_options = SanitizeOptions(options, nullptr, options);
return GetMutableCFOptionsMap(sanitized_options);
}
#endif // ROCKSDB_LITE
};
// RocksDB lite don't support dynamic options.
#ifndef ROCKSDB_LITE
TEST_F(DBOptionsTest, GetLatestOptions) {
// GetOptions should be able to get latest option changed by SetOptions.
Options options;
options.create_if_missing = true;
Random rnd(228);
Reopen(options);
CreateColumnFamilies({"foo"}, options);
ReopenWithColumnFamilies({"default", "foo"}, options);
auto options_default = GetRandomizedMutableCFOptionsMap(&rnd);
auto options_foo = GetRandomizedMutableCFOptionsMap(&rnd);
ASSERT_OK(dbfull()->SetOptions(handles_[0], options_default));
ASSERT_OK(dbfull()->SetOptions(handles_[1], options_foo));
ASSERT_EQ(options_default,
GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[0])));
ASSERT_EQ(options_foo,
GetMutableCFOptionsMap(dbfull()->GetOptions(handles_[1])));
}
TEST_F(DBOptionsTest, EnableAutoCompactionAndTriggerStall) {
const std::string kValue(1024, 'v');
for (int method_type = 0; method_type < 2; method_type++) {

3
db/db_test.cc
Unescape View File

@ -2806,8 +2806,7 @@ class ModelDB : public DB {
virtual Env* GetEnv() const override { return nullptr; }
using DB::GetOptions;
virtual const Options& GetOptions(
ColumnFamilyHandle* column_family) const override {
virtual Options GetOptions(ColumnFamilyHandle* column_family) const override {
return options_;
}

2
include/rocksdb/convenience.h
Unescape View File

@ -282,7 +282,7 @@ Status GetStringFromDBOptions(std::string* opts_str,
const std::string& delimiter = "; ");
Status GetStringFromColumnFamilyOptions(std::string* opts_str,
const ColumnFamilyOptions& db_options,
const ColumnFamilyOptions& cf_options,
const std::string& delimiter = "; ");
Status GetStringFromCompressionType(std::string* compression_str,

5
include/rocksdb/db.h
Unescape View File

@ -696,9 +696,8 @@ class DB {
// column family, the options provided when calling DB::Open() or
// DB::CreateColumnFamily() will have been "sanitized" and transformed
// in an implementation-defined manner.
virtual const Options& GetOptions(ColumnFamilyHandle* column_family)
const = 0;
virtual const Options& GetOptions() const {
virtual Options GetOptions(ColumnFamilyHandle* column_family) const = 0;
virtual Options GetOptions() const {
return GetOptions(DefaultColumnFamily());
}

3
include/rocksdb/utilities/stackable_db.h
Unescape View File

@ -219,8 +219,7 @@ class StackableDB : public DB {
}
using DB::GetOptions;
virtual const Options& GetOptions(ColumnFamilyHandle* column_family) const
override {
virtual Options GetOptions(ColumnFamilyHandle* column_family) const override {
return db_->GetOptions(column_family);
}

115
util/options_helper.cc
Unescape View File

@ -26,6 +26,65 @@
namespace rocksdb {
ColumnFamilyOptions BuildColumnFamilyOptions(
const ColumnFamilyOptions& options,
const MutableCFOptions& mutable_cf_options) {
ColumnFamilyOptions cf_opts(options);
// Memtable related options
cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size;
cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number;
cf_opts.arena_block_size = mutable_cf_options.arena_block_size;
cf_opts.memtable_prefix_bloom_size_ratio =
mutable_cf_options.memtable_prefix_bloom_size_ratio;
cf_opts.memtable_huge_page_size = mutable_cf_options.memtable_huge_page_size;
cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges;
cf_opts.inplace_update_num_locks =
mutable_cf_options.inplace_update_num_locks;
// Compaction related options
cf_opts.disable_auto_compactions =
mutable_cf_options.disable_auto_compactions;
cf_opts.level0_file_num_compaction_trigger =
mutable_cf_options.level0_file_num_compaction_trigger;
cf_opts.level0_slowdown_writes_trigger =
mutable_cf_options.level0_slowdown_writes_trigger;
cf_opts.level0_stop_writes_trigger =
mutable_cf_options.level0_stop_writes_trigger;
cf_opts.max_compaction_bytes = mutable_cf_options.max_compaction_bytes;
cf_opts.target_file_size_base = mutable_cf_options.target_file_size_base;
cf_opts.target_file_size_multiplier =
mutable_cf_options.target_file_size_multiplier;
cf_opts.max_bytes_for_level_base =
mutable_cf_options.max_bytes_for_level_base;
cf_opts.max_bytes_for_level_multiplier =
mutable_cf_options.max_bytes_for_level_multiplier;
cf_opts.max_bytes_for_level_multiplier_additional.clear();
for (auto value :
mutable_cf_options.max_bytes_for_level_multiplier_additional) {
cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value);
}
cf_opts.verify_checksums_in_compaction =
mutable_cf_options.verify_checksums_in_compaction;
// Misc options
cf_opts.max_sequential_skip_in_iterations =
mutable_cf_options.max_sequential_skip_in_iterations;
cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks;
cf_opts.report_bg_io_stats = mutable_cf_options.report_bg_io_stats;
cf_opts.compression = mutable_cf_options.compression;
cf_opts.min_partial_merge_operands =
mutable_cf_options.min_partial_merge_operands;
cf_opts.table_factory = options.table_factory;
// TODO(yhchiang): find some way to handle the following derived options
// * max_file_size
return cf_opts;
}
#ifndef ROCKSDB_LITE
bool isSpecialChar(const char c) {
if (c == '\\' || c == '#' || c == ':' || c == '\r' || c == '\n') {
@ -1367,60 +1426,6 @@ Status GetTableFactoryFromMap(
return Status::OK();
}
ColumnFamilyOptions BuildColumnFamilyOptions(
const Options& options, const MutableCFOptions& mutable_cf_options) {
ColumnFamilyOptions cf_opts(options);
// Memtable related options
cf_opts.write_buffer_size = mutable_cf_options.write_buffer_size;
cf_opts.max_write_buffer_number = mutable_cf_options.max_write_buffer_number;
cf_opts.arena_block_size = mutable_cf_options.arena_block_size;
cf_opts.memtable_prefix_bloom_size_ratio =
mutable_cf_options.memtable_prefix_bloom_size_ratio;
cf_opts.memtable_huge_page_size = mutable_cf_options.memtable_huge_page_size;
cf_opts.max_successive_merges = mutable_cf_options.max_successive_merges;
cf_opts.inplace_update_num_locks =
mutable_cf_options.inplace_update_num_locks;
// Compaction related options
cf_opts.disable_auto_compactions =
mutable_cf_options.disable_auto_compactions;
cf_opts.level0_file_num_compaction_trigger =
mutable_cf_options.level0_file_num_compaction_trigger;
cf_opts.level0_slowdown_writes_trigger =
mutable_cf_options.level0_slowdown_writes_trigger;
cf_opts.level0_stop_writes_trigger =
mutable_cf_options.level0_stop_writes_trigger;
cf_opts.max_compaction_bytes = mutable_cf_options.max_compaction_bytes;
cf_opts.target_file_size_base = mutable_cf_options.target_file_size_base;
cf_opts.target_file_size_multiplier =
mutable_cf_options.target_file_size_multiplier;
cf_opts.max_bytes_for_level_base =
mutable_cf_options.max_bytes_for_level_base;
cf_opts.max_bytes_for_level_multiplier =
mutable_cf_options.max_bytes_for_level_multiplier;
cf_opts.max_bytes_for_level_multiplier_additional.clear();
for (auto value :
mutable_cf_options.max_bytes_for_level_multiplier_additional) {
cf_opts.max_bytes_for_level_multiplier_additional.emplace_back(value);
}
cf_opts.verify_checksums_in_compaction =
mutable_cf_options.verify_checksums_in_compaction;
// Misc options
cf_opts.max_sequential_skip_in_iterations =
mutable_cf_options.max_sequential_skip_in_iterations;
cf_opts.paranoid_file_checks = mutable_cf_options.paranoid_file_checks;
cf_opts.report_bg_io_stats = mutable_cf_options.report_bg_io_stats;
cf_opts.table_factory = options.table_factory;
// TODO(yhchiang): find some way to handle the following derived options
// * max_file_size
return cf_opts;
}
#endif // !ROCKSDB_LITE
} // namespace rocksdb

13
util/options_helper.h
Unescape View File

@ -14,9 +14,14 @@
#include "rocksdb/table.h"
#include "util/cf_options.h"
#ifndef ROCKSDB_LITE
namespace rocksdb {
ColumnFamilyOptions BuildColumnFamilyOptions(
const ColumnFamilyOptions& ioptions,
const MutableCFOptions& mutable_cf_options);
#ifndef ROCKSDB_LITE
// Returns true if the input char "c" is considered as a special character
// that will be escaped when EscapeOptionString() is called.
//
@ -68,9 +73,6 @@ Status GetTableFactoryFromMap(
Status GetStringFromTableFactory(std::string* opts_str, const TableFactory* tf,
const std::string& delimiter = "; ");
ColumnFamilyOptions BuildColumnFamilyOptions(
const Options& options, const MutableCFOptions& mutable_cf_options);
enum class OptionType {
kBoolean,
kInt,
@ -672,6 +674,7 @@ static std::unordered_map<std::string, InfoLogLevel> info_log_level_string_map =
{"FATAL_LEVEL", InfoLogLevel::FATAL_LEVEL},
{"HEADER_LEVEL", InfoLogLevel::HEADER_LEVEL}};
#endif // !ROCKSDB_LITE
} // namespace rocksdb
#endif // !ROCKSDB_LITE

3
utilities/backupable/backupable_db_test.cc
Unescape View File

@ -57,8 +57,7 @@ class DummyDB : public StackableDB {
}
using DB::GetOptions;
virtual const Options& GetOptions(ColumnFamilyHandle* column_family) const
override {
virtual Options GetOptions(ColumnFamilyHandle* column_family) const override {
return options_;
}

Write Preview
Loading…
Cancel
Save