Make trash-to-DB size ratio limit configurable

Summary:
Allow users to configure the trash-to-DB size ratio limit, so
that ratelimits for deletes can be enforced even when larger portions of
the database are being deleted.
Closes https://github.com/facebook/rocksdb/pull/3158

Differential Revision: D6304897

Pulled By: gdavidsson

fbshipit-source-id: a28dd13059ebab7d4171b953ed91ce383a84d6b3

HISTORY.md

@@ -11,6 +11,7 @@
 * API call `DB::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum)` was added, users who wish to preserve deletes are expected to periodically call this function to advance the cutoff seqnum (all deletes made before this seqnum can be dropped by DB). It's user responsibility to figure out how to advance the seqnum in the way so the tombstones are kept for the desired period of time, yet are eventually processed in time and don't eat up too much space.
 * `ReadOptions::iter_start_seqnum` was added; if set to something > 0 user will see 2 changes in iterators behavior 1) only keys written with sequence larger than this parameter would be returned and 2) the `Slice` returned by iter->key() now points to the the memory that keep User-oriented representation of the internal key, rather than user key. New struct `FullKey` was added to represent internal keys, along with a new helper function `ParseFullKey(const Slice& internal_key, FullKey* result);`.
 * Deprecate trash_dir param in NewSstFileManager, right now we will rename deleted files to <name>.trash instead of moving them to trash directory
+* Allow setting a custom trash/DB size ratio limit in the SstFileManager, after which files that are to be scheduled for deletion are deleted immediately, regardless of any delete ratelimit.
 * Return an error on write if write_options.sync = true and write_options.disableWAL = true to warn user of inconsistent options. Previously we will not write to WAL and not respecting the sync options in this case.
 
 ### New Features

db/db_sst_test.cc

@@ -330,11 +330,11 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   int64_t rate_bytes_per_sec = 1024 * 10;  // 10 Kbs / Sec
   Status s;
   options.sst_file_manager.reset(
-      NewSstFileManager(env_, nullptr, "", 0, false, &s));
+      NewSstFileManager(env_, nullptr, "", 0, false, &s, 0));
   ASSERT_OK(s);
   options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec);
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
+  sfm->delete_scheduler()->SetMaxTrashDBRatio(1.1);
 
   ASSERT_OK(TryReopen(options));
   // Create 4 files in L0
@@ -396,10 +396,11 @@ TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) {
   int64_t rate_bytes_per_sec = 1024 * 1024;  // 1 Mb / Sec
   Status s;
   options.sst_file_manager.reset(
-      NewSstFileManager(env_, nullptr, "", rate_bytes_per_sec, false, &s));
+      NewSstFileManager(env_, nullptr, "", rate_bytes_per_sec, false, &s,
+                        /* max_trash_db_ratio= */ 1.1));
+
   ASSERT_OK(s);
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
 
   DestroyAndReopen(options);
 
@@ -459,7 +460,7 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
   options.disable_auto_compactions = true;
   options.env = env_;
   options.sst_file_manager.reset(
-      NewSstFileManager(env_, nullptr, "", 0, false, &s));
+      NewSstFileManager(env_, nullptr, "", 0, false, &s, 0));
   ASSERT_OK(s);
   DestroyAndReopen(options);
 
@@ -477,7 +478,7 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
 
   sfm->SetDeleteRateBytesPerSecond(1024 * 1024);
-  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
+  sfm->delete_scheduler()->SetMaxTrashDBRatio(1.1);
   ASSERT_OK(DestroyDB(dbname_, options));
   sfm->WaitForEmptyTrash();
   // We have deleted the 4 sst files in the delete_scheduler

include/rocksdb/sst_file_manager.h

@@ -57,6 +57,14 @@ class SstFileManager {
   // zero means disable delete rate limiting and delete files immediately
   // thread-safe
   virtual void SetDeleteRateBytesPerSecond(int64_t delete_rate) = 0;
+
+  // Return trash/DB size ratio where new files will be deleted immediately
+  // thread-safe
+  virtual double GetMaxTrashDBRatio() = 0;
+
+  // Update trash/DB size ratio where new files will be deleted immediately
+  // thread-safe
+  virtual void SetMaxTrashDBRatio(double ratio) = 0;
 };
 
 // Create a new SstFileManager that can be shared among multiple RocksDB
@@ -75,9 +83,13 @@ class SstFileManager {
 //    if user provide trash_dir we will schedule deletes for files in the dir
 // @param status: If not nullptr, status will contain any errors that happened
 //    during creating the missing trash_dir or deleting existing files in trash.
+// @param max_trash_db_ratio: If the trash size constitutes for more than this
+//    fraction of the total DB size we will start deleting new files passed to
+//    DeleteScheduler immediately
 extern SstFileManager* NewSstFileManager(
     Env* env, std::shared_ptr<Logger> info_log = nullptr,
     std::string trash_dir = "", int64_t rate_bytes_per_sec = 0,
-    bool delete_existing_trash = true, Status* status = nullptr);
+    bool delete_existing_trash = true, Status* status = nullptr,
+    double max_trash_db_ratio = 0.25);
 
 }  // namespace rocksdb

util/delete_scheduler.cc

@@ -21,7 +21,8 @@ namespace rocksdb {
 
 DeleteScheduler::DeleteScheduler(Env* env, int64_t rate_bytes_per_sec,
                                  Logger* info_log,
-                                 SstFileManagerImpl* sst_file_manager)
+                                 SstFileManagerImpl* sst_file_manager,
+                                 double max_trash_db_ratio)
     : env_(env),
       total_trash_size_(0),
       rate_bytes_per_sec_(rate_bytes_per_sec),
@@ -29,8 +30,10 @@ DeleteScheduler::DeleteScheduler(Env* env, int64_t rate_bytes_per_sec,
       closing_(false),
       cv_(&mu_),
       info_log_(info_log),
-      sst_file_manager_(sst_file_manager) {
+      sst_file_manager_(sst_file_manager),
+      max_trash_db_ratio_(max_trash_db_ratio) {
   assert(sst_file_manager != nullptr);
+  assert(max_trash_db_ratio >= 0);
   bg_thread_.reset(
       new port::Thread(&DeleteScheduler::BackgroundEmptyTrash, this));
 }
@@ -50,7 +53,7 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path) {
   Status s;
   if (rate_bytes_per_sec_.load() <= 0 ||
       total_trash_size_.load() >
-          sst_file_manager_->GetTotalSize() * max_trash_db_ratio_) {
+          sst_file_manager_->GetTotalSize() * max_trash_db_ratio_.load()) {
     // Rate limiting is disabled or trash size makes up more than
     // max_trash_db_ratio_ (default 25%) of the total DB size
     TEST_SYNC_POINT("DeleteScheduler::DeleteFile");

util/delete_scheduler.h

@@ -33,7 +33,8 @@ class SstFileManagerImpl;
 class DeleteScheduler {
  public:
   DeleteScheduler(Env* env, int64_t rate_bytes_per_sec, Logger* info_log,
-                  SstFileManagerImpl* sst_file_manager);
+                  SstFileManagerImpl* sst_file_manager,
+                  double max_trash_db_ratio);
 
   ~DeleteScheduler();
 
@@ -42,7 +43,7 @@ class DeleteScheduler {
 
   // Set delete rate limit in bytes per second
   void SetRateBytesPerSecond(int64_t bytes_per_sec) {
-    return rate_bytes_per_sec_.store(bytes_per_sec);
+    rate_bytes_per_sec_.store(bytes_per_sec);
   }
 
   // Mark file as trash directory and schedule it's deletion
@@ -58,9 +59,15 @@ class DeleteScheduler {
 
   uint64_t GetTotalTrashSize() { return total_trash_size_.load(); }
 
-  void TEST_SetMaxTrashDBRatio(double r) {
+  // Return trash/DB size ratio where new files will be deleted immediately
+  double GetMaxTrashDBRatio() {
+    return max_trash_db_ratio_.load();
+  }
+
+  // Update trash/DB size ratio where new files will be deleted immediately
+  void SetMaxTrashDBRatio(double r) {
     assert(r >= 0);
-    max_trash_db_ratio_ = r;
+    max_trash_db_ratio_.store(r);
   }
 
   static const std::string kTrashExtension;
@@ -105,9 +112,10 @@ class DeleteScheduler {
   InstrumentedMutex file_move_mu_;
   Logger* info_log_;
   SstFileManagerImpl* sst_file_manager_;
-  // If the trash size constitutes for more than 25% of the total DB size
-  // we will start deleting new files passed to DeleteScheduler immediately
-  double max_trash_db_ratio_ = 0.25;
+  // If the trash size constitutes for more than this fraction of the total DB
+  // size we will start deleting new files passed to DeleteScheduler
+  // immediately
+  std::atomic<double> max_trash_db_ratio_;
   static const uint64_t kMicrosInSecond = 1000 * 1000LL;
 };
 

util/delete_scheduler_test.cc

@@ -83,12 +83,13 @@ class DeleteSchedulerTest : public testing::Test {
   }
 
   void NewDeleteScheduler() {
+    // Tests in this file are for DeleteScheduler component and dont create any
+    // DBs, so we need to set max_trash_db_ratio to 100% (instead of default
+    // 25%)
     sst_file_mgr_.reset(
-        new SstFileManagerImpl(env_, nullptr, rate_bytes_per_sec_));
+        new SstFileManagerImpl(env_, nullptr, rate_bytes_per_sec_,
+                               /* max_trash_db_ratio= */ 1.1));
     delete_scheduler_ = sst_file_mgr_->delete_scheduler();
-    // Tests in this file are for DeleteScheduler component and dont create any
-    // DBs, so we need to use set this value to 100% (instead of default 25%)
-    delete_scheduler_->TEST_SetMaxTrashDBRatio(1.1);
   }
 
   Env* env_;
@@ -517,7 +518,7 @@ TEST_F(DeleteSchedulerTest, ImmediateDeleteOn25PercDBSize) {
   rate_bytes_per_sec_ = 1;  // 1 byte per sec (very slow trash delete)
 
   NewDeleteScheduler();
-  delete_scheduler_->TEST_SetMaxTrashDBRatio(0.25);
+  delete_scheduler_->SetMaxTrashDBRatio(0.25);
 
   std::vector<std::string> generated_files;
   for (int i = 0; i < num_files; i++) {

util/sst_file_manager_impl.cc

@@ -17,12 +17,14 @@ namespace rocksdb {
 
 #ifndef ROCKSDB_LITE
 SstFileManagerImpl::SstFileManagerImpl(Env* env, std::shared_ptr<Logger> logger,
-                                       int64_t rate_bytes_per_sec)
+                                       int64_t rate_bytes_per_sec,
+                                       double max_trash_db_ratio)
     : env_(env),
       logger_(logger),
       total_files_size_(0),
       max_allowed_space_(0),
-      delete_scheduler_(env, rate_bytes_per_sec, logger.get(), this) {}
+      delete_scheduler_(env, rate_bytes_per_sec, logger.get(), this,
+                        max_trash_db_ratio) {}
 
 SstFileManagerImpl::~SstFileManagerImpl() {}
 
@@ -93,6 +95,14 @@ void SstFileManagerImpl::SetDeleteRateBytesPerSecond(int64_t delete_rate) {
   return delete_scheduler_.SetRateBytesPerSecond(delete_rate);
 }
 
+double SstFileManagerImpl::GetMaxTrashDBRatio() {
+  return delete_scheduler_.GetMaxTrashDBRatio();
+}
+
+void SstFileManagerImpl::SetMaxTrashDBRatio(double r) {
+  return delete_scheduler_.SetMaxTrashDBRatio(r);
+}
+
 Status SstFileManagerImpl::ScheduleFileDeletion(const std::string& file_path) {
   return delete_scheduler_.DeleteFile(file_path);
 }
@@ -128,9 +138,11 @@ void SstFileManagerImpl::OnDeleteFileImpl(const std::string& file_path) {
 SstFileManager* NewSstFileManager(Env* env, std::shared_ptr<Logger> info_log,
                                   std::string trash_dir,
                                   int64_t rate_bytes_per_sec,
-                                  bool delete_existing_trash, Status* status) {
+                                  bool delete_existing_trash, Status* status,
+                                  double max_trash_db_ratio) {
   SstFileManagerImpl* res =
-      new SstFileManagerImpl(env, info_log, rate_bytes_per_sec);
+      new SstFileManagerImpl(env, info_log, rate_bytes_per_sec,
+                             max_trash_db_ratio);
 
   // trash_dir is deprecated and not needed anymore, but if user passed it
   // we will still remove files in it.
@@ -166,10 +178,11 @@ SstFileManager* NewSstFileManager(Env* env, std::shared_ptr<Logger> info_log,
 SstFileManager* NewSstFileManager(Env* env, std::shared_ptr<Logger> info_log,
                                   std::string trash_dir,
                                   int64_t rate_bytes_per_sec,
-                                  bool delete_existing_trash, Status* status) {
+                                  bool delete_existing_trash, Status* status,
+                                  double max_trash_db_ratio) {
   if (status) {
     *status =
-        Status::NotSupported("SstFileManager is not supported in ROCKSDB_LITE");
+      Status::NotSupported("SstFileManager is not supported in ROCKSDB_LITE");
   }
   return nullptr;
 }
@@ -177,4 +190,3 @@ SstFileManager* NewSstFileManager(Env* env, std::shared_ptr<Logger> info_log,
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb
-

util/sst_file_manager_impl.h

@@ -25,7 +25,8 @@ class Logger;
 class SstFileManagerImpl : public SstFileManager {
  public:
   explicit SstFileManagerImpl(Env* env, std::shared_ptr<Logger> logger,
-                              int64_t rate_bytes_per_sec);
+                              int64_t rate_bytes_per_sec,
+                              double max_trash_db_ratio);
 
   ~SstFileManagerImpl();
 
@@ -67,6 +68,12 @@ class SstFileManagerImpl : public SstFileManager {
   // Update the delete rate limit in bytes per second.
   virtual void SetDeleteRateBytesPerSecond(int64_t delete_rate) override;
 
+  // Return trash/DB size ratio where new files will be deleted immediately
+  virtual double GetMaxTrashDBRatio() override;
+
+  // Update trash/DB size ratio where new files will be deleted immediately
+  virtual void SetMaxTrashDBRatio(double ratio) override;
+
   // Mark file as trash and schedule it's deletion.
   virtual Status ScheduleFileDeletion(const std::string& file_path);