Made delete_obsolete_files_period_micros option dynamic

Summary:
Made delete_obsolete_files_period_micros option dynamic. It can be updating using DB::SetDBOptions().
Closes https://github.com/facebook/rocksdb/pull/1595

Differential Revision: D4246569

Pulled By: tonek

fbshipit-source-id: d23f560

HISTORY.md

@@ -4,6 +4,7 @@
 * Options.level0_stop_writes_trigger default value changes from 24 to 32.
 * New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
 * Removed flashcache support.
+* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
 
 ## 5.0.0 (11/17/2016)
 ### Public API Change

db/db_impl.cc

@@ -332,9 +332,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
       num_running_flushes_(0),
       bg_purge_scheduled_(0),
       disable_delete_obsolete_files_(0),
-      delete_obsolete_files_next_run_(
+      delete_obsolete_files_last_run_(env_->NowMicros()),
-          env_->NowMicros() +
-          immutable_db_options_.delete_obsolete_files_period_micros),
       last_stats_dump_time_microsec_(0),
       next_job_id_(1),
       has_unpersisted_data_(false),
@@ -743,7 +741,7 @@ void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
 // Otherwise, gets obsolete files from VersionSet.
 // no_full_scan = true -- never do the full scan using GetChildren()
 // force = false -- don't force the full scan, except every
-//  immutable_db_options_.delete_obsolete_files_period_micros
+//  mutable_db_options_.delete_obsolete_files_period_micros
 // force = true -- force the full scan
 void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
                                bool no_full_scan) {
@@ -760,15 +758,15 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   if (no_full_scan) {
     doing_the_full_scan = false;
   } else if (force ||
-             immutable_db_options_.delete_obsolete_files_period_micros == 0) {
+             mutable_db_options_.delete_obsolete_files_period_micros == 0) {
     doing_the_full_scan = true;
   } else {
     const uint64_t now_micros = env_->NowMicros();
-    if (delete_obsolete_files_next_run_ < now_micros) {
+    if ((delete_obsolete_files_last_run_ +
+         mutable_db_options_.delete_obsolete_files_period_micros) <
+        now_micros) {
       doing_the_full_scan = true;
-      delete_obsolete_files_next_run_ =
+      delete_obsolete_files_last_run_ = now_micros;
-          now_micros +
-          immutable_db_options_.delete_obsolete_files_period_micros;
     }
   }
 

db/db_impl.h

@@ -972,8 +972,9 @@ class DBImpl : public DB {
   // without any synchronization
   int disable_delete_obsolete_files_;
 
-  // next time when we should run DeleteObsoleteFiles with full scan
+  // last time when DeleteObsoleteFiles with full scan was executed. Originaly
-  uint64_t delete_obsolete_files_next_run_;
+  // initialized with startup time.
+  uint64_t delete_obsolete_files_last_run_;
 
   // last time stats were dumped to LOG
   std::atomic<uint64_t> last_stats_dump_time_microsec_;

db/db_options_test.cc

@@ -316,6 +316,47 @@ TEST_F(DBOptionsTest, MaxTotalWalSizeChange) {
   }
 }
 
+static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) {
+  dbfull->TEST_LockMutex();
+  JobContext job_context(0);
+  dbfull->FindObsoleteFiles(&job_context, false);
+  ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty());
+  job_context.Clean();
+  dbfull->TEST_UnlockMutex();
+}
+
+TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
+  SpecialEnv env(Env::Default());
+  env.time_elapse_only_sleep_ = true;
+  Options options;
+  options.env = &env;
+  options.create_if_missing = true;
+  ASSERT_OK(TryReopen(options));
+
+  // Verify that candidate files set is empty when no full scan requested.
+  assert_candidate_files_empty(dbfull(), true);
+
+  ASSERT_OK(
+      dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}}));
+
+  // After delete_obsolete_files_period_micros updated to 0, the next call
+  // to FindObsoleteFiles should make a full scan
+  assert_candidate_files_empty(dbfull(), false);
+
+  ASSERT_OK(
+      dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}}));
+
+  assert_candidate_files_empty(dbfull(), true);
+
+  env.addon_time_.store(20);
+  assert_candidate_files_empty(dbfull(), true);
+
+  env.addon_time_.store(21);
+  assert_candidate_files_empty(dbfull(), false);
+
+  Close();
+}
+
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

util/db_options.cc

@@ -39,8 +39,6 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
       db_paths(options.db_paths),
       db_log_dir(options.db_log_dir),
       wal_dir(options.wal_dir),
-      delete_obsolete_files_period_micros(
-          options.delete_obsolete_files_period_micros),
       max_subcompactions(options.max_subcompactions),
       max_background_flushes(options.max_background_flushes),
       max_log_file_size(options.max_log_file_size),
@@ -139,8 +137,6 @@ void ImmutableDBOptions::Dump(Logger* log) const {
          wal_dir.c_str());
   Header(log, "               Options.table_cache_numshardbits: %d",
          table_cache_numshardbits);
-  Header(log, "    Options.delete_obsolete_files_period_micros: %" PRIu64,
-         delete_obsolete_files_period_micros);
   Header(log, "                     Options.max_subcompactions: %" PRIu32,
          max_subcompactions);
   Header(log, "                 Options.max_background_flushes: %d",
@@ -222,14 +218,17 @@ MutableDBOptions::MutableDBOptions()
       max_background_compactions(1),
       avoid_flush_during_shutdown(false),
       delayed_write_rate(2 * 1024U * 1024U),
-      max_total_wal_size(0) {}
+      max_total_wal_size(0),
+      delete_obsolete_files_period_micros(6ULL * 60 * 60 * 1000000) {}
 
 MutableDBOptions::MutableDBOptions(const DBOptions& options)
     : base_background_compactions(options.base_background_compactions),
       max_background_compactions(options.max_background_compactions),
       avoid_flush_during_shutdown(options.avoid_flush_during_shutdown),
       delayed_write_rate(options.delayed_write_rate),
-      max_total_wal_size(options.max_total_wal_size) {}
+      max_total_wal_size(options.max_total_wal_size),
+      delete_obsolete_files_period_micros(
+          options.delete_obsolete_files_period_micros) {}
 
 void MutableDBOptions::Dump(Logger* log) const {
   Header(log, "            Options.base_background_compactions: %d",
@@ -242,6 +241,9 @@ void MutableDBOptions::Dump(Logger* log) const {
          delayed_write_rate);
   Header(log, "            Options.max_total_wal_size: %" PRIu64,
          max_total_wal_size);
+  Header(log,
+         "            Options.delete_obsolete_files_period_micros: %" PRIu64,
+         delete_obsolete_files_period_micros);
 }
 
 }  // namespace rocksdb

util/db_options.h

@@ -35,7 +35,6 @@ struct ImmutableDBOptions {
   std::vector<DbPath> db_paths;
   std::string db_log_dir;
   std::string wal_dir;
-  uint64_t delete_obsolete_files_period_micros;
   uint32_t max_subcompactions;
   int max_background_flushes;
   size_t max_log_file_size;
@@ -95,6 +94,7 @@ struct MutableDBOptions {
   bool avoid_flush_during_shutdown;
   uint64_t delayed_write_rate;
   uint64_t max_total_wal_size;
+  uint64_t delete_obsolete_files_period_micros;
 };
 
 }  // namespace rocksdb

util/options_helper.cc

@@ -53,7 +53,7 @@ DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
   options.db_log_dir = immutable_db_options.db_log_dir;
   options.wal_dir = immutable_db_options.wal_dir;
   options.delete_obsolete_files_period_micros =
-      immutable_db_options.delete_obsolete_files_period_micros;
+      mutable_db_options.delete_obsolete_files_period_micros;
   options.base_background_compactions =
       mutable_db_options.base_background_compactions;
   options.max_background_compactions =

util/options_helper.h

@@ -302,7 +302,8 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
       offsetof(struct MutableDBOptions, delayed_write_rate)}},
     {"delete_obsolete_files_period_micros",
      {offsetof(struct DBOptions, delete_obsolete_files_period_micros),
-      OptionType::kUInt64T, OptionVerificationType::kNormal, false, 0}},
+      OptionType::kUInt64T, OptionVerificationType::kNormal, true,
+      offsetof(struct MutableDBOptions, delete_obsolete_files_period_micros)}},
     {"max_manifest_file_size",
      {offsetof(struct DBOptions, max_manifest_file_size), OptionType::kUInt64T,
       OptionVerificationType::kNormal, false, 0}},