NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

SetOptions() to return status and also add it to StackableDB

Browse Source 
Summary: as title

Test Plan: ./db_test

Reviewers: sdong, yhchiang, rven, igor

Reviewed By: igor

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D28269
main
Lei Jin 10 years ago
parent b1267750fb
commit fd24ae9d05
10 changed files with 50 additions and 41 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 10
      db/column_family.cc
  2. 2
      db/column_family.h
  3. 14
      db/db_impl.cc
  4. 2
      db/db_impl.h
  5. 34
      db/db_test.cc
  6. 6
      include/rocksdb/db.h
  7. 7
      include/rocksdb/utilities/stackable_db.h
  8. 2
      tools/db_stress.cc
  9. 11
      util/options_helper.cc
  10. 3
      util/options_helper.h

10
db/column_family.cc
Unescape View File

@ -544,16 +544,16 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
} }
} }
bool ColumnFamilyData::SetOptions( Status ColumnFamilyData::SetOptions(
const std::unordered_map<std::string, std::string>& options_map) { const std::unordered_map<std::string, std::string>& options_map) {
MutableCFOptions new_mutable_cf_options; MutableCFOptions new_mutable_cf_options;
if (GetMutableOptionsFromStrings(mutable_cf_options_, options_map, Status s = GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
&new_mutable_cf_options)) { &new_mutable_cf_options);
if (s.ok()) {
mutable_cf_options_ = new_mutable_cf_options; mutable_cf_options_ = new_mutable_cf_options;
mutable_cf_options_.RefreshDerivedOptions(ioptions_); mutable_cf_options_.RefreshDerivedOptions(ioptions_);
return true;
} }
return false; return s;
} }
ColumnFamilySet::ColumnFamilySet(const std::string& dbname, ColumnFamilySet::ColumnFamilySet(const std::string& dbname,

2
db/column_family.h
Unescape View File

@ -187,7 +187,7 @@ class ColumnFamilyData {
return &mutable_cf_options_; return &mutable_cf_options_;
} }
// REQUIRES: DB mutex held // REQUIRES: DB mutex held
bool SetOptions( Status SetOptions(
const std::unordered_map<std::string, std::string>& options_map); const std::unordered_map<std::string, std::string>& options_map);
InternalStats* internal_stats() { return internal_stats_.get(); } InternalStats* internal_stats() { return internal_stats_.get(); }

14
db/db_impl.cc
Unescape View File

@ -1142,23 +1142,23 @@ Status DBImpl::CompactRange(ColumnFamilyHandle* column_family,
return s; return s;
} }
bool DBImpl::SetOptions(ColumnFamilyHandle* column_family, Status DBImpl::SetOptions(ColumnFamilyHandle* column_family,
const std::unordered_map<std::string, std::string>& options_map) { const std::unordered_map<std::string, std::string>& options_map) {
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd(); auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
if (options_map.empty()) { if (options_map.empty()) {
Log(InfoLogLevel::WARN_LEVEL, Log(InfoLogLevel::WARN_LEVEL,
db_options_.info_log, "SetOptions() on column family [%s], empty input", db_options_.info_log, "SetOptions() on column family [%s], empty input",
cfd->GetName().c_str()); cfd->GetName().c_str());
return false; return Status::InvalidArgument("empty input");
} }
MutableCFOptions new_options; MutableCFOptions new_options;
bool succeed = false; Status s;
{ {
MutexLock l(&mutex_); MutexLock l(&mutex_);
if (cfd->SetOptions(options_map)) { s = cfd->SetOptions(options_map);
if (s.ok()) {
new_options = *cfd->GetLatestMutableCFOptions(); new_options = *cfd->GetLatestMutableCFOptions();
succeed = true;
} }
} }
@ -1169,7 +1169,7 @@ bool DBImpl::SetOptions(ColumnFamilyHandle* column_family,
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log, Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
"%s: %s\n", o.first.c_str(), o.second.c_str()); "%s: %s\n", o.first.c_str(), o.second.c_str());
} }
if (succeed) { if (s.ok()) {
Log(InfoLogLevel::INFO_LEVEL, Log(InfoLogLevel::INFO_LEVEL,
db_options_.info_log, "[%s] SetOptions succeeded", db_options_.info_log, "[%s] SetOptions succeeded",
cfd->GetName().c_str()); cfd->GetName().c_str());
@ -1178,7 +1178,7 @@ bool DBImpl::SetOptions(ColumnFamilyHandle* column_family,
Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log, Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
"[%s] SetOptions failed", cfd->GetName().c_str()); "[%s] SetOptions failed", cfd->GetName().c_str());
} }
return succeed; return s;
} }
// return the same level if it cannot be moved // return the same level if it cannot be moved

2
db/db_impl.h
Unescape View File

@ -115,7 +115,7 @@ class DBImpl : public DB {
uint32_t target_path_id = 0); uint32_t target_path_id = 0);
using DB::SetOptions; using DB::SetOptions;
bool SetOptions(ColumnFamilyHandle* column_family, Status SetOptions(ColumnFamilyHandle* column_family,
const std::unordered_map<std::string, std::string>& options_map); const std::unordered_map<std::string, std::string>& options_map);
using DB::NumberLevels; using DB::NumberLevels;

34
db/db_test.cc
Unescape View File

@ -8453,7 +8453,7 @@ TEST(DBTest, DynamicMemtableOptions) {
ASSERT_EQ(NumTableFilesAtLevel(0), 0); ASSERT_EQ(NumTableFilesAtLevel(0), 0);
// Increase buffer size // Increase buffer size
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"write_buffer_size", "131072"}, {"write_buffer_size", "131072"},
})); }));
@ -8495,7 +8495,7 @@ TEST(DBTest, DynamicMemtableOptions) {
sleeping_task_low1.WaitUntilDone(); sleeping_task_low1.WaitUntilDone();
// Increase // Increase
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"max_write_buffer_number", "8"}, {"max_write_buffer_number", "8"},
})); }));
// Clean up memtable and L0 // Clean up memtable and L0
@ -8514,7 +8514,7 @@ TEST(DBTest, DynamicMemtableOptions) {
sleeping_task_low2.WaitUntilDone(); sleeping_task_low2.WaitUntilDone();
// Decrease // Decrease
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"max_write_buffer_number", "4"}, {"max_write_buffer_number", "4"},
})); }));
// Clean up memtable and L0 // Clean up memtable and L0
@ -8593,7 +8593,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// Writing to 64KB L0 files should trigger a compaction. Since these // Writing to 64KB L0 files should trigger a compaction. Since these
// 2 L0 files have the same key range, compaction merge them and should // 2 L0 files have the same key range, compaction merge them and should
// result in 2 32KB L1 files. // result in 2 32KB L1 files.
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"level0_file_num_compaction_trigger", "2"}, {"level0_file_num_compaction_trigger", "2"},
{"target_file_size_base", std::to_string(k32KB) } {"target_file_size_base", std::to_string(k32KB) }
})); }));
@ -8615,7 +8615,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// Increase level base size to 256KB and write enough data that will // Increase level base size to 256KB and write enough data that will
// fill L1 and L2. L1 size should be around 256KB while L2 size should be // fill L1 and L2. L1 size should be around 256KB while L2 size should be
// around 256KB x 4. // around 256KB x 4.
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"max_bytes_for_level_base", std::to_string(k1MB) } {"max_bytes_for_level_base", std::to_string(k1MB) }
})); }));
@ -8636,7 +8636,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// max_bytes_for_level_base. Now, reduce both mulitplier and level base, // max_bytes_for_level_base. Now, reduce both mulitplier and level base,
// After filling enough data that can fit in L1 - L3, we should see L1 size // After filling enough data that can fit in L1 - L3, we should see L1 size
// reduces to 128KB from 256KB which was asserted previously. Same for L2. // reduces to 128KB from 256KB which was asserted previously. Same for L2.
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"max_bytes_for_level_multiplier", "2"}, {"max_bytes_for_level_multiplier", "2"},
{"max_bytes_for_level_base", std::to_string(k128KB) } {"max_bytes_for_level_base", std::to_string(k128KB) }
})); }));
@ -8678,7 +8678,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// Now reduce level0_stop_writes_trigger to 6. Clear up memtables and L0. // Now reduce level0_stop_writes_trigger to 6. Clear up memtables and L0.
// Block compaction thread again. Perform the put and memtable flushes // Block compaction thread again. Perform the put and memtable flushes
// until we see timeout after 6 memtable flushes. // until we see timeout after 6 memtable flushes.
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"level0_stop_writes_trigger", "6"} {"level0_stop_writes_trigger", "6"}
})); }));
dbfull()->CompactRange(nullptr, nullptr); dbfull()->CompactRange(nullptr, nullptr);
@ -8703,7 +8703,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// 4 L0 files and compaction should be triggered. If auto compaction is // 4 L0 files and compaction should be triggered. If auto compaction is
// disabled, then TEST_WaitForCompact will be waiting for nothing. Number of // disabled, then TEST_WaitForCompact will be waiting for nothing. Number of
// L0 files do not change after the call. // L0 files do not change after the call.
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"disable_auto_compactions", "true"} {"disable_auto_compactions", "true"}
})); }));
dbfull()->CompactRange(nullptr, nullptr); dbfull()->CompactRange(nullptr, nullptr);
@ -8719,7 +8719,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// Enable auto compaction and perform the same test, # of L0 files should be // Enable auto compaction and perform the same test, # of L0 files should be
// reduced after compaction. // reduced after compaction.
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"disable_auto_compactions", "false"} {"disable_auto_compactions", "false"}
})); }));
dbfull()->CompactRange(nullptr, nullptr); dbfull()->CompactRange(nullptr, nullptr);
@ -8737,7 +8737,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// First change max_bytes_for_level_base to a big value and populate // First change max_bytes_for_level_base to a big value and populate
// L1 - L3. Then thrink max_bytes_for_level_base and disable auto compaction // L1 - L3. Then thrink max_bytes_for_level_base and disable auto compaction
// at the same time, we should see some level with score greater than 2. // at the same time, we should see some level with score greater than 2.
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"max_bytes_for_level_base", std::to_string(k1MB) } {"max_bytes_for_level_base", std::to_string(k1MB) }
})); }));
// writing 40 x 64KB = 10 x 256KB // writing 40 x 64KB = 10 x 256KB
@ -8754,7 +8754,7 @@ TEST(DBTest, DynamicCompactionOptions) {
SizeAtLevel(3) < 4 * k1MB * 1.2)); SizeAtLevel(3) < 4 * k1MB * 1.2));
// Reduce max_bytes_for_level_base and disable compaction at the same time // Reduce max_bytes_for_level_base and disable compaction at the same time
// This should cause score to increase // This should cause score to increase
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"disable_auto_compactions", "true"}, {"disable_auto_compactions", "true"},
{"max_bytes_for_level_base", "65536"}, {"max_bytes_for_level_base", "65536"},
})); }));
@ -8769,7 +8769,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// Enfoce hard rate limit. Now set hard_rate_limit to 2, // Enfoce hard rate limit. Now set hard_rate_limit to 2,
// we should start to see put delay (1000 us) and timeout as a result // we should start to see put delay (1000 us) and timeout as a result
// (L0 score is not regulated by this limit). // (L0 score is not regulated by this limit).
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"hard_rate_limit", "2"} {"hard_rate_limit", "2"}
})); }));
ASSERT_OK(Put(Key(count), RandomString(&rnd, 1024))); ASSERT_OK(Put(Key(count), RandomString(&rnd, 1024)));
@ -8781,7 +8781,7 @@ TEST(DBTest, DynamicCompactionOptions) {
ASSERT_TRUE(Put(Key(count), RandomString(&rnd, 1024), wo).IsTimedOut()); ASSERT_TRUE(Put(Key(count), RandomString(&rnd, 1024), wo).IsTimedOut());
// Lift the limit and no timeout // Lift the limit and no timeout
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"hard_rate_limit", "100"} {"hard_rate_limit", "100"}
})); }));
dbfull()->TEST_FlushMemTable(true); dbfull()->TEST_FlushMemTable(true);
@ -8807,7 +8807,7 @@ TEST(DBTest, DynamicCompactionOptions) {
ASSERT_TRUE(Put("max_mem_compaction_level_key", ASSERT_TRUE(Put("max_mem_compaction_level_key",
RandomString(&rnd, 8)).ok()); RandomString(&rnd, 8)).ok());
// Set new value and it becomes effective in this flush // Set new value and it becomes effective in this flush
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"max_mem_compaction_level", "1"} {"max_mem_compaction_level", "1"}
})); }));
dbfull()->TEST_FlushMemTable(true); dbfull()->TEST_FlushMemTable(true);
@ -8818,7 +8818,7 @@ TEST(DBTest, DynamicCompactionOptions) {
ASSERT_TRUE(Put("max_mem_compaction_level_key", ASSERT_TRUE(Put("max_mem_compaction_level_key",
RandomString(&rnd, 8)).ok()); RandomString(&rnd, 8)).ok());
// Set new value and it becomes effective in this flush // Set new value and it becomes effective in this flush
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"max_mem_compaction_level", "0"} {"max_mem_compaction_level", "0"}
})); }));
dbfull()->TEST_FlushMemTable(true); dbfull()->TEST_FlushMemTable(true);
@ -8994,7 +8994,7 @@ TEST(DBTest, DynamicMiscOptions) {
// No reseek // No reseek
assert_reseek_count(100, 0); assert_reseek_count(100, 0);
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"max_sequential_skip_in_iterations", "4"} {"max_sequential_skip_in_iterations", "4"}
})); }));
// Clear memtable and make new option effective // Clear memtable and make new option effective
@ -9002,7 +9002,7 @@ TEST(DBTest, DynamicMiscOptions) {
// Trigger reseek // Trigger reseek
assert_reseek_count(200, 1); assert_reseek_count(200, 1);
ASSERT_TRUE(dbfull()->SetOptions({ ASSERT_OK(dbfull()->SetOptions({
{"max_sequential_skip_in_iterations", "16"} {"max_sequential_skip_in_iterations", "16"}
})); }));
// Clear memtable and make new option effective // Clear memtable and make new option effective

6
include/rocksdb/db.h
Unescape View File

@ -359,11 +359,11 @@ class DB {
return CompactRange(DefaultColumnFamily(), begin, end, reduce_level, return CompactRange(DefaultColumnFamily(), begin, end, reduce_level,
target_level, target_path_id); target_level, target_path_id);
} }
virtual bool SetOptions(ColumnFamilyHandle* column_family, virtual Status SetOptions(ColumnFamilyHandle* column_family,
const std::unordered_map<std::string, std::string>& new_options) { const std::unordered_map<std::string, std::string>& new_options) {
return true; return Status::NotSupported("Not implemented");
} }
virtual bool SetOptions( virtual Status SetOptions(
const std::unordered_map<std::string, std::string>& new_options) { const std::unordered_map<std::string, std::string>& new_options) {
return SetOptions(DefaultColumnFamily(), new_options); return SetOptions(DefaultColumnFamily(), new_options);
} }

7
include/rocksdb/utilities/stackable_db.h
Unescape View File

@ -3,6 +3,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once #pragma once
#include <string>
#include "rocksdb/db.h" #include "rocksdb/db.h"
namespace rocksdb { namespace rocksdb {
@ -203,6 +204,12 @@ class StackableDB : public DB {
return db_->GetDbIdentity(identity); return db_->GetDbIdentity(identity);
} }
using DB::SetOptions;
virtual Status SetOptions(
const std::unordered_map<std::string, std::string>& new_options) override {
return db_->SetOptions(new_options);
}
using DB::GetPropertiesOfAllTables; using DB::GetPropertiesOfAllTables;
virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family, virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
TablePropertiesCollection* props) { TablePropertiesCollection* props) {

2
tools/db_stress.cc
Unescape View File

@ -1298,7 +1298,7 @@ class StressTest {
return s; return s;
} }
bool SetOptions(ThreadState* thread) { Status SetOptions(ThreadState* thread) {
assert(FLAGS_set_options_one_in > 0); assert(FLAGS_set_options_one_in > 0);
std::unordered_map<std::string, std::string> opts; std::unordered_map<std::string, std::string> opts;
std::string name = options_index_[ std::string name = options_index_[

11
util/options_helper.cc
Unescape View File

@ -165,7 +165,7 @@ bool ParseMiscOptions(const std::string& name, const std::string& value,
return true; return true;
} }
bool GetMutableOptionsFromStrings( Status GetMutableOptionsFromStrings(
const MutableCFOptions& base_options, const MutableCFOptions& base_options,
const std::unordered_map<std::string, std::string>& options_map, const std::unordered_map<std::string, std::string>& options_map,
MutableCFOptions* new_options) { MutableCFOptions* new_options) {
@ -177,13 +177,14 @@ bool GetMutableOptionsFromStrings(
} else if (ParseCompactionOptions(o.first, o.second, new_options)) { } else if (ParseCompactionOptions(o.first, o.second, new_options)) {
} else if (ParseMiscOptions(o.first, o.second, new_options)) { } else if (ParseMiscOptions(o.first, o.second, new_options)) {
} else { } else {
return false; return Status::InvalidArgument(
"unsupported dynamic option: " + o.first);
} }
} }
} catch (std::exception) { } catch (std::exception& e) {
return false; return Status::InvalidArgument("error parsing " + std::string(e.what()));
} }
return true; return Status::OK();
} }
namespace { namespace {

3
util/options_helper.h
Unescape View File

@ -7,10 +7,11 @@
#include <string> #include <string>
#include "util/mutable_cf_options.h" #include "util/mutable_cf_options.h"
#include "rocksdb/status.h"
namespace rocksdb { namespace rocksdb {
bool GetMutableOptionsFromStrings( Status GetMutableOptionsFromStrings(
const MutableCFOptions& base_options, const MutableCFOptions& base_options,
const std::unordered_map<std::string, std::string>& options_map, const std::unordered_map<std::string, std::string>& options_map,
MutableCFOptions* new_options); MutableCFOptions* new_options);

Write Preview
Loading…
Cancel
Save