Made delete_obsolete_files_period_micros option dynamic

Summary:
Made delete_obsolete_files_period_micros option dynamic. It can be updating using DB::SetDBOptions().
Closes https://github.com/facebook/rocksdb/pull/1595

Differential Revision: D4246569

Pulled By: tonek

fbshipit-source-id: d23f560
main
Anton Safonov 8 years ago committed by Facebook Github Bot
parent edde954e7b
commit 9053fe2a5c
  1. 1
      HISTORY.md
  2. 16
      db/db_impl.cc
  3. 5
      db/db_impl.h
  4. 41
      db/db_options_test.cc
  5. 14
      util/db_options.cc
  6. 2
      util/db_options.h
  7. 2
      util/options_helper.cc
  8. 3
      util/options_helper.h

@ -4,6 +4,7 @@
* Options.level0_stop_writes_trigger default value changes from 24 to 32. * Options.level0_stop_writes_trigger default value changes from 24 to 32.
* New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys. * New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
* Removed flashcache support. * Removed flashcache support.
* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
## 5.0.0 (11/17/2016) ## 5.0.0 (11/17/2016)
### Public API Change ### Public API Change

@ -332,9 +332,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
num_running_flushes_(0), num_running_flushes_(0),
bg_purge_scheduled_(0), bg_purge_scheduled_(0),
disable_delete_obsolete_files_(0), disable_delete_obsolete_files_(0),
delete_obsolete_files_next_run_( delete_obsolete_files_last_run_(env_->NowMicros()),
env_->NowMicros() +
immutable_db_options_.delete_obsolete_files_period_micros),
last_stats_dump_time_microsec_(0), last_stats_dump_time_microsec_(0),
next_job_id_(1), next_job_id_(1),
has_unpersisted_data_(false), has_unpersisted_data_(false),
@ -743,7 +741,7 @@ void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
// Otherwise, gets obsolete files from VersionSet. // Otherwise, gets obsolete files from VersionSet.
// no_full_scan = true -- never do the full scan using GetChildren() // no_full_scan = true -- never do the full scan using GetChildren()
// force = false -- don't force the full scan, except every // force = false -- don't force the full scan, except every
// immutable_db_options_.delete_obsolete_files_period_micros // mutable_db_options_.delete_obsolete_files_period_micros
// force = true -- force the full scan // force = true -- force the full scan
void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force, void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
bool no_full_scan) { bool no_full_scan) {
@ -760,15 +758,15 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
if (no_full_scan) { if (no_full_scan) {
doing_the_full_scan = false; doing_the_full_scan = false;
} else if (force || } else if (force ||
immutable_db_options_.delete_obsolete_files_period_micros == 0) { mutable_db_options_.delete_obsolete_files_period_micros == 0) {
doing_the_full_scan = true; doing_the_full_scan = true;
} else { } else {
const uint64_t now_micros = env_->NowMicros(); const uint64_t now_micros = env_->NowMicros();
if (delete_obsolete_files_next_run_ < now_micros) { if ((delete_obsolete_files_last_run_ +
mutable_db_options_.delete_obsolete_files_period_micros) <
now_micros) {
doing_the_full_scan = true; doing_the_full_scan = true;
delete_obsolete_files_next_run_ = delete_obsolete_files_last_run_ = now_micros;
now_micros +
immutable_db_options_.delete_obsolete_files_period_micros;
} }
} }

@ -972,8 +972,9 @@ class DBImpl : public DB {
// without any synchronization // without any synchronization
int disable_delete_obsolete_files_; int disable_delete_obsolete_files_;
// next time when we should run DeleteObsoleteFiles with full scan // last time when DeleteObsoleteFiles with full scan was executed. Originaly
uint64_t delete_obsolete_files_next_run_; // initialized with startup time.
uint64_t delete_obsolete_files_last_run_;
// last time stats were dumped to LOG // last time stats were dumped to LOG
std::atomic<uint64_t> last_stats_dump_time_microsec_; std::atomic<uint64_t> last_stats_dump_time_microsec_;

@ -316,6 +316,47 @@ TEST_F(DBOptionsTest, MaxTotalWalSizeChange) {
} }
} }
static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) {
dbfull->TEST_LockMutex();
JobContext job_context(0);
dbfull->FindObsoleteFiles(&job_context, false);
ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty());
job_context.Clean();
dbfull->TEST_UnlockMutex();
}
TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
SpecialEnv env(Env::Default());
env.time_elapse_only_sleep_ = true;
Options options;
options.env = &env;
options.create_if_missing = true;
ASSERT_OK(TryReopen(options));
// Verify that candidate files set is empty when no full scan requested.
assert_candidate_files_empty(dbfull(), true);
ASSERT_OK(
dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}}));
// After delete_obsolete_files_period_micros updated to 0, the next call
// to FindObsoleteFiles should make a full scan
assert_candidate_files_empty(dbfull(), false);
ASSERT_OK(
dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}}));
assert_candidate_files_empty(dbfull(), true);
env.addon_time_.store(20);
assert_candidate_files_empty(dbfull(), true);
env.addon_time_.store(21);
assert_candidate_files_empty(dbfull(), false);
Close();
}
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
} // namespace rocksdb } // namespace rocksdb

@ -39,8 +39,6 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
db_paths(options.db_paths), db_paths(options.db_paths),
db_log_dir(options.db_log_dir), db_log_dir(options.db_log_dir),
wal_dir(options.wal_dir), wal_dir(options.wal_dir),
delete_obsolete_files_period_micros(
options.delete_obsolete_files_period_micros),
max_subcompactions(options.max_subcompactions), max_subcompactions(options.max_subcompactions),
max_background_flushes(options.max_background_flushes), max_background_flushes(options.max_background_flushes),
max_log_file_size(options.max_log_file_size), max_log_file_size(options.max_log_file_size),
@ -139,8 +137,6 @@ void ImmutableDBOptions::Dump(Logger* log) const {
wal_dir.c_str()); wal_dir.c_str());
Header(log, " Options.table_cache_numshardbits: %d", Header(log, " Options.table_cache_numshardbits: %d",
table_cache_numshardbits); table_cache_numshardbits);
Header(log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
delete_obsolete_files_period_micros);
Header(log, " Options.max_subcompactions: %" PRIu32, Header(log, " Options.max_subcompactions: %" PRIu32,
max_subcompactions); max_subcompactions);
Header(log, " Options.max_background_flushes: %d", Header(log, " Options.max_background_flushes: %d",
@ -222,14 +218,17 @@ MutableDBOptions::MutableDBOptions()
max_background_compactions(1), max_background_compactions(1),
avoid_flush_during_shutdown(false), avoid_flush_during_shutdown(false),
delayed_write_rate(2 * 1024U * 1024U), delayed_write_rate(2 * 1024U * 1024U),
max_total_wal_size(0) {} max_total_wal_size(0),
delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000) {}
MutableDBOptions::MutableDBOptions(const DBOptions& options) MutableDBOptions::MutableDBOptions(const DBOptions& options)
: base_background_compactions(options.base_background_compactions), : base_background_compactions(options.base_background_compactions),
max_background_compactions(options.max_background_compactions), max_background_compactions(options.max_background_compactions),
avoid_flush_during_shutdown(options.avoid_flush_during_shutdown), avoid_flush_during_shutdown(options.avoid_flush_during_shutdown),
delayed_write_rate(options.delayed_write_rate), delayed_write_rate(options.delayed_write_rate),
max_total_wal_size(options.max_total_wal_size) {} max_total_wal_size(options.max_total_wal_size),
delete_obsolete_files_period_micros(
options.delete_obsolete_files_period_micros) {}
void MutableDBOptions::Dump(Logger* log) const { void MutableDBOptions::Dump(Logger* log) const {
Header(log, " Options.base_background_compactions: %d", Header(log, " Options.base_background_compactions: %d",
@ -242,6 +241,9 @@ void MutableDBOptions::Dump(Logger* log) const {
delayed_write_rate); delayed_write_rate);
Header(log, " Options.max_total_wal_size: %" PRIu64, Header(log, " Options.max_total_wal_size: %" PRIu64,
max_total_wal_size); max_total_wal_size);
Header(log,
" Options.delete_obsolete_files_period_micros: %" PRIu64,
delete_obsolete_files_period_micros);
} }
} // namespace rocksdb } // namespace rocksdb

@ -35,7 +35,6 @@ struct ImmutableDBOptions {
std::vector<DbPath> db_paths; std::vector<DbPath> db_paths;
std::string db_log_dir; std::string db_log_dir;
std::string wal_dir; std::string wal_dir;
uint64_t delete_obsolete_files_period_micros;
uint32_t max_subcompactions; uint32_t max_subcompactions;
int max_background_flushes; int max_background_flushes;
size_t max_log_file_size; size_t max_log_file_size;
@ -95,6 +94,7 @@ struct MutableDBOptions {
bool avoid_flush_during_shutdown; bool avoid_flush_during_shutdown;
uint64_t delayed_write_rate; uint64_t delayed_write_rate;
uint64_t max_total_wal_size; uint64_t max_total_wal_size;
uint64_t delete_obsolete_files_period_micros;
}; };
} // namespace rocksdb } // namespace rocksdb

@ -53,7 +53,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
options.db_log_dir = immutable_db_options.db_log_dir; options.db_log_dir = immutable_db_options.db_log_dir;
options.wal_dir = immutable_db_options.wal_dir; options.wal_dir = immutable_db_options.wal_dir;
options.delete_obsolete_files_period_micros = options.delete_obsolete_files_period_micros =
immutable_db_options.delete_obsolete_files_period_micros; mutable_db_options.delete_obsolete_files_period_micros;
options.base_background_compactions = options.base_background_compactions =
mutable_db_options.base_background_compactions; mutable_db_options.base_background_compactions;
options.max_background_compactions = options.max_background_compactions =

@ -302,7 +302,8 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
offsetof(struct MutableDBOptions, delayed_write_rate)}}, offsetof(struct MutableDBOptions, delayed_write_rate)}},
{"delete_obsolete_files_period_micros", {"delete_obsolete_files_period_micros",
{offsetof(struct DBOptions, delete_obsolete_files_period_micros), {offsetof(struct DBOptions, delete_obsolete_files_period_micros),
OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}}, OptionType::kUInt64T, OptionVerificationType::kNormal, true,
offsetof(struct MutableDBOptions, delete_obsolete_files_period_micros)}},
{"max_manifest_file_size", {"max_manifest_file_size",
{offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T, {offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T,
OptionVerificationType::kNormal, false, 0}}, OptionVerificationType::kNormal, false, 0}},

Loading…
Cancel
Save