Add avoid_flush_during_shutdown DB option

Summary:
Add avoid_flush_during_shutdown DB option.
Closes https://github.com/facebook/rocksdb/pull/1451

Differential Revision: D4108643

Pulled By: yiwu-arbug

fbshipit-source-id: abdaf4d
main
Yi Wu 8 years ago committed by Facebook Github Bot
parent 2b16d664cb
commit 437942e481
  1. 3
      HISTORY.md
  2. 3
      db/db_impl.cc
  3. 22
      db/db_options_test.cc
  4. 9
      include/rocksdb/options.h
  5. 9
      util/db_options.cc
  6. 1
      util/db_options.h
  7. 6
      util/options.cc
  8. 2
      util/options_helper.cc
  9. 6
      util/options_helper.h
  10. 3
      util/options_settable_test.cc
  11. 2
      util/testutil.cc

@ -3,6 +3,9 @@
### Public API Change ### Public API Change
* Options::max_bytes_for_level_multiplier is now a double along with all getters and setters. * Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
### New Features
* Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable.
## 4.13.0 (10/18/2016) ## 4.13.0 (10/18/2016)
### Public API Change ### Public API Change
* DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference. * DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.

@ -375,7 +375,8 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
InstrumentedMutexLock l(&mutex_); InstrumentedMutexLock l(&mutex_);
if (!shutting_down_.load(std::memory_order_acquire) && if (!shutting_down_.load(std::memory_order_acquire) &&
has_unpersisted_data_) { has_unpersisted_data_ &&
!mutable_db_options_.avoid_flush_during_shutdown) {
for (auto cfd : *versions_->GetColumnFamilySet()) { for (auto cfd : *versions_->GetColumnFamilySet()) {
if (!cfd->IsDropped() && !cfd->mem()->IsEmpty()) { if (!cfd->IsDropped() && !cfd->mem()->IsEmpty()) {
cfd->Ref(); cfd->Ref();

@ -256,6 +256,28 @@ TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed()); ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed());
} }
TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) {
Options options;
options.create_if_missing = true;
options.disable_auto_compactions = true;
WriteOptions write_without_wal;
write_without_wal.disableWAL = true;
ASSERT_FALSE(options.avoid_flush_during_shutdown);
DestroyAndReopen(options);
ASSERT_OK(Put("foo", "v1", write_without_wal));
Reopen(options);
ASSERT_EQ("v1", Get("foo"));
ASSERT_EQ("1", FilesPerLevel());
DestroyAndReopen(options);
ASSERT_OK(Put("foo", "v2", write_without_wal));
ASSERT_OK(dbfull()->SetDBOptions({{"avoid_flush_during_shutdown", "true"}}));
Reopen(options);
ASSERT_EQ("NOT_FOUND", Get("foo"));
ASSERT_EQ("", FilesPerLevel());
}
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
} // namespace rocksdb } // namespace rocksdb

@ -1354,6 +1354,15 @@ struct DBOptions {
// //
// DEFAULT: false // DEFAULT: false
bool avoid_flush_during_recovery; bool avoid_flush_during_recovery;
// By default RocksDB will flush all memtables on DB close if there are
// unpersisted data (i.e. with WAL disabled) The flush can be skip to speedup
// DB close. Unpersisted data WILL BE LOST.
//
// DEFAULT: false
//
// Dynamically changeable through SetDBOptions() API.
bool avoid_flush_during_shutdown;
}; };
// Options to control the behavior of a database (passed to DB::Open) // Options to control the behavior of a database (passed to DB::Open)

@ -224,17 +224,22 @@ void ImmutableDBOptions::Dump(Logger* log) const {
} }
MutableDBOptions::MutableDBOptions() MutableDBOptions::MutableDBOptions()
: base_background_compactions(1), max_background_compactions(1) {} : base_background_compactions(1),
max_background_compactions(1),
avoid_flush_during_shutdown(false) {}
MutableDBOptions::MutableDBOptions(const DBOptions& options) MutableDBOptions::MutableDBOptions(const DBOptions& options)
: base_background_compactions(options.base_background_compactions), : base_background_compactions(options.base_background_compactions),
max_background_compactions(options.max_background_compactions) {} max_background_compactions(options.max_background_compactions),
avoid_flush_during_shutdown(options.avoid_flush_during_shutdown) {}
void MutableDBOptions::Dump(Logger* log) const { void MutableDBOptions::Dump(Logger* log) const {
Header(log, " Options.base_background_compactions: %d", Header(log, " Options.base_background_compactions: %d",
base_background_compactions); base_background_compactions);
Header(log, " Options.max_background_compactions: %d", Header(log, " Options.max_background_compactions: %d",
max_background_compactions); max_background_compactions);
Header(log, " Options.avoid_flush_during_shutdown: %d",
avoid_flush_during_shutdown);
} }
} // namespace rocksdb } // namespace rocksdb

@ -94,6 +94,7 @@ struct MutableDBOptions {
int base_background_compactions; int base_background_compactions;
int max_background_compactions; int max_background_compactions;
bool avoid_flush_during_shutdown;
}; };
} // namespace rocksdb } // namespace rocksdb

@ -228,7 +228,8 @@ DBOptions::DBOptions()
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
fail_if_options_file_error(false), fail_if_options_file_error(false),
dump_malloc_stats(false), dump_malloc_stats(false),
avoid_flush_during_recovery(false) { avoid_flush_during_recovery(false),
avoid_flush_during_shutdown(false) {
} }
DBOptions::DBOptions(const Options& options) DBOptions::DBOptions(const Options& options)
@ -301,7 +302,8 @@ DBOptions::DBOptions(const Options& options)
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
fail_if_options_file_error(options.fail_if_options_file_error), fail_if_options_file_error(options.fail_if_options_file_error),
dump_malloc_stats(options.dump_malloc_stats), dump_malloc_stats(options.dump_malloc_stats),
avoid_flush_during_recovery(options.avoid_flush_during_recovery) { avoid_flush_during_recovery(options.avoid_flush_during_recovery),
avoid_flush_during_shutdown(options.avoid_flush_during_shutdown) {
} }
static const char* const access_hints[] = { static const char* const access_hints[] = {

@ -118,6 +118,8 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
options.dump_malloc_stats = immutable_db_options.dump_malloc_stats; options.dump_malloc_stats = immutable_db_options.dump_malloc_stats;
options.avoid_flush_during_recovery = options.avoid_flush_during_recovery =
immutable_db_options.avoid_flush_during_recovery; immutable_db_options.avoid_flush_during_recovery;
options.avoid_flush_during_shutdown =
mutable_db_options.avoid_flush_during_shutdown;
return options; return options;
} }

@ -343,7 +343,11 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
OptionVerificationType::kNormal, false, 0}}, OptionVerificationType::kNormal, false, 0}},
{"avoid_flush_during_recovery", {"avoid_flush_during_recovery",
{offsetof(struct DBOptions, avoid_flush_during_recovery), {offsetof(struct DBOptions, avoid_flush_during_recovery),
OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}}}; OptionType::kBoolean, OptionVerificationType::kNormal, false, 0}},
{"avoid_flush_during_shutdown",
{offsetof(struct DBOptions, avoid_flush_during_shutdown),
OptionType::kBoolean, OptionVerificationType::kNormal, true,
offsetof(struct MutableDBOptions, avoid_flush_during_shutdown)}}};
static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = { static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
/* not yet supported /* not yet supported

@ -288,7 +288,8 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
"info_log_level=DEBUG_LEVEL;" "info_log_level=DEBUG_LEVEL;"
"dump_malloc_stats=false;" "dump_malloc_stats=false;"
"allow_2pc=false;" "allow_2pc=false;"
"avoid_flush_during_recovery=false;", "avoid_flush_during_recovery=false;"
"avoid_flush_during_shutdown=false;",
new_options)); new_options));
ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions), ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),

@ -253,6 +253,8 @@ void RandomInitDBOptions(DBOptions* db_opt, Random* rnd) {
db_opt->use_adaptive_mutex = rnd->Uniform(2); db_opt->use_adaptive_mutex = rnd->Uniform(2);
db_opt->use_fsync = rnd->Uniform(2); db_opt->use_fsync = rnd->Uniform(2);
db_opt->recycle_log_file_num = rnd->Uniform(2); db_opt->recycle_log_file_num = rnd->Uniform(2);
db_opt->avoid_flush_during_recovery = rnd->Uniform(2);
db_opt->avoid_flush_during_shutdown = rnd->Uniform(2);
// int options // int options
db_opt->max_background_compactions = rnd->Uniform(100); db_opt->max_background_compactions = rnd->Uniform(100);

Loading…
Cancel
Save