NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Made delete_obsolete_files_period_micros option dynamic

Browse Source 
Summary:
Made delete_obsolete_files_period_micros option dynamic. It can be updating using DB::SetDBOptions().
Closes https://github.com/facebook/rocksdb/pull/1595

Differential Revision: D4246569

Pulled By: tonek

fbshipit-source-id: d23f560
main
Anton Safonov 8 years ago committed by Facebook Github Bot
parent edde954e7b
commit 9053fe2a5c
8 changed files with 64 additions and 20 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      HISTORY.md
  2. 16
      db/db_impl.cc
  3. 5
      db/db_impl.h
  4. 41
      db/db_options_test.cc
  5. 14
      util/db_options.cc
  6. 2
      util/db_options.h
  7. 2
      util/options_helper.cc
  8. 3
      util/options_helper.h

1
HISTORY.md
Unescape View File

@ -4,6 +4,7 @@
* Options.level0_stop_writes_trigger default value changes from 24 to 32.
* New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
* Removed flashcache support.
* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
## 5.0.0 (11/17/2016)
### Public API Change

16
db/db_impl.cc
Unescape View File

@ -332,9 +332,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
num_running_flushes_(0),
bg_purge_scheduled_(0),
disable_delete_obsolete_files_(0),
delete_obsolete_files_next_run_(
env_->NowMicros() +
immutable_db_options_.delete_obsolete_files_period_micros),
delete_obsolete_files_last_run_(env_->NowMicros()),
last_stats_dump_time_microsec_(0),
next_job_id_(1),
has_unpersisted_data_(false),
@ -743,7 +741,7 @@ void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
// Otherwise, gets obsolete files from VersionSet.
// no_full_scan = true -- never do the full scan using GetChildren()
// force = false -- don't force the full scan, except every
// immutable_db_options_.delete_obsolete_files_period_micros
// mutable_db_options_.delete_obsolete_files_period_micros
// force = true -- force the full scan
void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
bool no_full_scan) {
@ -760,15 +758,15 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
if (no_full_scan) {
doing_the_full_scan = false;
} else if (force ||
immutable_db_options_.delete_obsolete_files_period_micros == 0) {
mutable_db_options_.delete_obsolete_files_period_micros == 0) {
doing_the_full_scan = true;
} else {
const uint64_t now_micros = env_->NowMicros();
if (delete_obsolete_files_next_run_ < now_micros) {
if ((delete_obsolete_files_last_run_ +
mutable_db_options_.delete_obsolete_files_period_micros) <
now_micros) {
doing_the_full_scan = true;
delete_obsolete_files_next_run_ =
now_micros +
immutable_db_options_.delete_obsolete_files_period_micros;
delete_obsolete_files_last_run_ = now_micros;
}
}

5
db/db_impl.h
Unescape View File

@ -972,8 +972,9 @@ class DBImpl : public DB {
// without any synchronization
int disable_delete_obsolete_files_;
// next time when we should run DeleteObsoleteFiles with full scan
uint64_t delete_obsolete_files_next_run_;
// last time when DeleteObsoleteFiles with full scan was executed. Originaly
// initialized with startup time.
uint64_t delete_obsolete_files_last_run_;
// last time stats were dumped to LOG
std::atomic<uint64_t> last_stats_dump_time_microsec_;

41
db/db_options_test.cc
Unescape View File

@ -316,6 +316,47 @@ TEST_F(DBOptionsTest, MaxTotalWalSizeChange) {
}
}
static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) {
dbfull->TEST_LockMutex();
JobContext job_context(0);
dbfull->FindObsoleteFiles(&job_context, false);
ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty());
job_context.Clean();
dbfull->TEST_UnlockMutex();
}
TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
SpecialEnv env(Env::Default());
env.time_elapse_only_sleep_ = true;
Options options;
options.env = &env;
options.create_if_missing = true;
ASSERT_OK(TryReopen(options));
// Verify that candidate files set is empty when no full scan requested.
assert_candidate_files_empty(dbfull(), true);
ASSERT_OK(
dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}}));
// After delete_obsolete_files_period_micros updated to 0, the next call
// to FindObsoleteFiles should make a full scan
assert_candidate_files_empty(dbfull(), false);
ASSERT_OK(
dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}}));
assert_candidate_files_empty(dbfull(), true);
env.addon_time_.store(20);
assert_candidate_files_empty(dbfull(), true);
env.addon_time_.store(21);
assert_candidate_files_empty(dbfull(), false);
Close();
}
#endif // ROCKSDB_LITE
} // namespace rocksdb

14
util/db_options.cc
Unescape View File

@ -39,8 +39,6 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
db_paths(options.db_paths),
db_log_dir(options.db_log_dir),
wal_dir(options.wal_dir),
delete_obsolete_files_period_micros(
options.delete_obsolete_files_period_micros),
max_subcompactions(options.max_subcompactions),
max_background_flushes(options.max_background_flushes),
max_log_file_size(options.max_log_file_size),
@ -139,8 +137,6 @@ void ImmutableDBOptions::Dump(Logger* log) const {
wal_dir.c_str());
Header(log, " Options.table_cache_numshardbits: %d",
table_cache_numshardbits);
Header(log, " Options.delete_obsolete_files_period_micros: %" PRIu64,
delete_obsolete_files_period_micros);
Header(log, " Options.max_subcompactions: %" PRIu32,
max_subcompactions);
Header(log, " Options.max_background_flushes: %d",
@ -222,14 +218,17 @@ MutableDBOptions::MutableDBOptions()
max_background_compactions(1),
avoid_flush_during_shutdown(false),
delayed_write_rate(2 * 1024U * 1024U),
max_total_wal_size(0) {}
max_total_wal_size(0),
delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000) {}
MutableDBOptions::MutableDBOptions(const DBOptions& options)
: base_background_compactions(options.base_background_compactions),
max_background_compactions(options.max_background_compactions),
avoid_flush_during_shutdown(options.avoid_flush_during_shutdown),
delayed_write_rate(options.delayed_write_rate),
max_total_wal_size(options.max_total_wal_size) {}
max_total_wal_size(options.max_total_wal_size),
delete_obsolete_files_period_micros(
options.delete_obsolete_files_period_micros) {}
void MutableDBOptions::Dump(Logger* log) const {
Header(log, " Options.base_background_compactions: %d",
@ -242,6 +241,9 @@ void MutableDBOptions::Dump(Logger* log) const {
delayed_write_rate);
Header(log, " Options.max_total_wal_size: %" PRIu64,
max_total_wal_size);
Header(log,
" Options.delete_obsolete_files_period_micros: %" PRIu64,
delete_obsolete_files_period_micros);
}
} // namespace rocksdb

2
util/db_options.h
Unescape View File

@ -35,7 +35,6 @@ struct ImmutableDBOptions {
std::vector<DbPath> db_paths;
std::string db_log_dir;
std::string wal_dir;
uint64_t delete_obsolete_files_period_micros;
uint32_t max_subcompactions;
int max_background_flushes;
size_t max_log_file_size;
@ -95,6 +94,7 @@ struct MutableDBOptions {
bool avoid_flush_during_shutdown;
uint64_t delayed_write_rate;
uint64_t max_total_wal_size;
uint64_t delete_obsolete_files_period_micros;
};
} // namespace rocksdb

2
util/options_helper.cc
Unescape View File

@ -53,7 +53,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
options.db_log_dir = immutable_db_options.db_log_dir;
options.wal_dir = immutable_db_options.wal_dir;
options.delete_obsolete_files_period_micros =
immutable_db_options.delete_obsolete_files_period_micros;
mutable_db_options.delete_obsolete_files_period_micros;
options.base_background_compactions =
mutable_db_options.base_background_compactions;
options.max_background_compactions =

3
util/options_helper.h
Unescape View File

@ -302,7 +302,8 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
offsetof(struct MutableDBOptions, delayed_write_rate)}},
{"delete_obsolete_files_period_micros",
{offsetof(struct DBOptions, delete_obsolete_files_period_micros),
OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
OptionType::kUInt64T, OptionVerificationType::kNormal, true,
offsetof(struct MutableDBOptions, delete_obsolete_files_period_micros)}},
{"max_manifest_file_size",
{offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T,
OptionVerificationType::kNormal, false, 0}},

Write Preview
Loading…
Cancel
Save