Summary: Introduce DeleteScheduler that allow enforcing a rate limit on file deletion Instead of deleting files immediately, files are moved to trash directory and deleted in a background thread that apply sleep penalty between deletes if needed. I have updated PurgeObsoleteFiles and PurgeObsoleteWALFiles to use the delete_scheduler instead of env_->DeleteFile Test Plan: added delete_scheduler_test existing unit tests Reviewers: kradhakrishnan, anthony, rven, yhchiang, sdong Reviewed By: sdong Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D43221main
Islam AbdelRahman
9 years ago
parent
102ac118b2
commit
c45a57b41e
@ -0,0 +1,62 @@ |
|||||||
|
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <map> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "rocksdb/status.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class Env; |
||||||
|
class Logger; |
||||||
|
|
||||||
|
// DeleteScheduler allow the DB to enforce a rate limit on file deletion,
|
||||||
|
// Instead of deleteing files immediately, files are moved to trash_dir
|
||||||
|
// and deleted in a background thread that apply sleep penlty between deletes
|
||||||
|
// if they are happening in a rate faster than rate_bytes_per_sec,
|
||||||
|
//
|
||||||
|
// Rate limiting can be turned off by setting rate_bytes_per_sec = 0, In this
|
||||||
|
// case DeleteScheduler will delete files immediately.
|
||||||
|
class DeleteScheduler { |
||||||
|
public: |
||||||
|
virtual ~DeleteScheduler() {} |
||||||
|
|
||||||
|
// Return delete rate limit in bytes per second
|
||||||
|
virtual int64_t GetRateBytesPerSecond() = 0; |
||||||
|
|
||||||
|
// Move file to trash directory and schedule it's deletion
|
||||||
|
virtual Status DeleteFile(const std::string& fname) = 0; |
||||||
|
|
||||||
|
// Return a map containing errors that happened in the background thread
|
||||||
|
// file_path => error status
|
||||||
|
virtual std::map<std::string, Status> GetBackgroundErrors() = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
// Create a new DeleteScheduler that can be shared among multiple RocksDB
|
||||||
|
// instances to control the file deletion rate.
|
||||||
|
//
|
||||||
|
// @env: Pointer to Env object, please see "rocksdb/env.h".
|
||||||
|
// @trash_dir: Path to the directory where deleted files will be moved into
|
||||||
|
// to be deleted in a background thread while applying rate limiting. If this
|
||||||
|
// directory dont exist, it will be created. This directory should not be
|
||||||
|
// used by any other process or any other DeleteScheduler.
|
||||||
|
// @rate_bytes_per_sec: How many bytes should be deleted per second, If this
|
||||||
|
// value is set to 1024 (1 Kb / sec) and we deleted a file of size 4 Kb
|
||||||
|
// in 1 second, we will wait for another 3 seconds before we delete other
|
||||||
|
// files, Set to 0 to disable rate limiting.
|
||||||
|
// @info_log: If not nullptr, info_log will be used to log errors.
|
||||||
|
// @delete_exisitng_trash: If set to true, the newly created DeleteScheduler
|
||||||
|
// will delete files that already exist in trash_dir.
|
||||||
|
// @status: If not nullptr, status will contain any errors that happened during
|
||||||
|
// creating the missing trash_dir or deleting existing files in trash.
|
||||||
|
extern DeleteScheduler* NewDeleteScheduler( |
||||||
|
Env* env, const std::string& trash_dir, int64_t rate_bytes_per_sec, |
||||||
|
std::shared_ptr<Logger> info_log = nullptr, |
||||||
|
bool delete_exisitng_trash = true, Status* status = nullptr); |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,228 @@ |
|||||||
|
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include <thread> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "port/port.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "util/delete_scheduler_impl.h" |
||||||
|
#include "util/mutexlock.h" |
||||||
|
#include "util/sync_point.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
DeleteSchedulerImpl::DeleteSchedulerImpl(Env* env, const std::string& trash_dir, |
||||||
|
int64_t rate_bytes_per_sec, |
||||||
|
std::shared_ptr<Logger> info_log) |
||||||
|
: env_(env), |
||||||
|
trash_dir_(trash_dir), |
||||||
|
rate_bytes_per_sec_(rate_bytes_per_sec), |
||||||
|
pending_files_(0), |
||||||
|
closing_(false), |
||||||
|
cv_(&mu_), |
||||||
|
info_log_(info_log) { |
||||||
|
if (rate_bytes_per_sec_ == 0) { |
||||||
|
// Rate limiting is disabled
|
||||||
|
bg_thread_.reset(); |
||||||
|
} else { |
||||||
|
bg_thread_.reset( |
||||||
|
new std::thread(&DeleteSchedulerImpl::BackgroundEmptyTrash, this)); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
DeleteSchedulerImpl::~DeleteSchedulerImpl() { |
||||||
|
{ |
||||||
|
MutexLock l(&mu_); |
||||||
|
closing_ = true; |
||||||
|
cv_.SignalAll(); |
||||||
|
} |
||||||
|
if (bg_thread_) { |
||||||
|
bg_thread_->join(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Status DeleteSchedulerImpl::DeleteFile(const std::string& file_path) { |
||||||
|
if (rate_bytes_per_sec_ == 0) { |
||||||
|
// Rate limiting is disabled
|
||||||
|
return env_->DeleteFile(file_path); |
||||||
|
} |
||||||
|
|
||||||
|
// Move file to trash
|
||||||
|
std::string path_in_trash; |
||||||
|
Status s = MoveToTrash(file_path, &path_in_trash); |
||||||
|
if (!s.ok()) { |
||||||
|
Log(InfoLogLevel::ERROR_LEVEL, info_log_, |
||||||
|
"Failed to move %s to trash directory (%s)", file_path.c_str(), |
||||||
|
trash_dir_.c_str()); |
||||||
|
return env_->DeleteFile(file_path); |
||||||
|
} |
||||||
|
|
||||||
|
// Add file to delete queue
|
||||||
|
{ |
||||||
|
MutexLock l(&mu_); |
||||||
|
queue_.push(path_in_trash); |
||||||
|
pending_files_++; |
||||||
|
if (pending_files_ == 1) { |
||||||
|
cv_.SignalAll(); |
||||||
|
} |
||||||
|
} |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
std::map<std::string, Status> DeleteSchedulerImpl::GetBackgroundErrors() { |
||||||
|
MutexLock l(&mu_); |
||||||
|
return bg_errors_; |
||||||
|
} |
||||||
|
|
||||||
|
Status DeleteSchedulerImpl::MoveToTrash(const std::string& file_path, |
||||||
|
std::string* path_in_trash) { |
||||||
|
Status s; |
||||||
|
// Figure out the name of the file in trash folder
|
||||||
|
size_t idx = file_path.rfind("/"); |
||||||
|
if (idx == std::string::npos || idx == file_path.size() - 1) { |
||||||
|
return Status::InvalidArgument("file_path is corrupted"); |
||||||
|
} |
||||||
|
*path_in_trash = trash_dir_ + file_path.substr(idx); |
||||||
|
std::string unique_suffix = ""; |
||||||
|
|
||||||
|
if (*path_in_trash == file_path) { |
||||||
|
// This file is already in trash
|
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
// TODO(tec) : Implement Env::RenameFileIfNotExist and remove
|
||||||
|
// file_move_mu mutex.
|
||||||
|
MutexLock l(&file_move_mu_); |
||||||
|
while (true) { |
||||||
|
s = env_->FileExists(*path_in_trash + unique_suffix); |
||||||
|
if (s.IsNotFound()) { |
||||||
|
// We found a path for our file in trash
|
||||||
|
*path_in_trash += unique_suffix; |
||||||
|
s = env_->RenameFile(file_path, *path_in_trash); |
||||||
|
break; |
||||||
|
} else if (s.ok()) { |
||||||
|
// Name conflict, generate new random suffix
|
||||||
|
unique_suffix = env_->GenerateUniqueId(); |
||||||
|
} else { |
||||||
|
// Error during FileExists call, we cannot continue
|
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
void DeleteSchedulerImpl::BackgroundEmptyTrash() { |
||||||
|
TEST_SYNC_POINT("DeleteSchedulerImpl::BackgroundEmptyTrash"); |
||||||
|
|
||||||
|
while (true) { |
||||||
|
MutexLock l(&mu_); |
||||||
|
while (queue_.empty() && !closing_) { |
||||||
|
cv_.Wait(); |
||||||
|
} |
||||||
|
|
||||||
|
if (closing_) { |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
// Delete all files in queue_
|
||||||
|
uint64_t start_time = env_->NowMicros(); |
||||||
|
uint64_t total_deleted_bytes = 0; |
||||||
|
while (!queue_.empty() && !closing_) { |
||||||
|
std::string path_in_trash = queue_.front(); |
||||||
|
queue_.pop(); |
||||||
|
|
||||||
|
// We dont need to hold the lock while deleting the file
|
||||||
|
mu_.Unlock(); |
||||||
|
uint64_t deleted_bytes = 0; |
||||||
|
// Delete file from trash and update total_penlty value
|
||||||
|
Status s = DeleteTrashFile(path_in_trash, &deleted_bytes); |
||||||
|
total_deleted_bytes += deleted_bytes; |
||||||
|
mu_.Lock(); |
||||||
|
|
||||||
|
if (!s.ok()) { |
||||||
|
bg_errors_[path_in_trash] = s; |
||||||
|
} |
||||||
|
|
||||||
|
// Apply penlty if necessary
|
||||||
|
uint64_t total_penlty = |
||||||
|
((total_deleted_bytes * kMicrosInSecond) / rate_bytes_per_sec_); |
||||||
|
while (!closing_ && !cv_.TimedWait(start_time + total_penlty)) {} |
||||||
|
|
||||||
|
pending_files_--; |
||||||
|
if (pending_files_ == 0) { |
||||||
|
// Unblock TEST_WaitForEmptyTrash since there are no more files waiting
|
||||||
|
// to be deleted
|
||||||
|
cv_.SignalAll(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Status DeleteSchedulerImpl::DeleteTrashFile(const std::string& path_in_trash, |
||||||
|
uint64_t* deleted_bytes) { |
||||||
|
uint64_t file_size; |
||||||
|
Status s = env_->GetFileSize(path_in_trash, &file_size); |
||||||
|
if (s.ok()) { |
||||||
|
TEST_SYNC_POINT("DeleteSchedulerImpl::DeleteTrashFile:DeleteFile"); |
||||||
|
s = env_->DeleteFile(path_in_trash); |
||||||
|
} |
||||||
|
|
||||||
|
if (!s.ok()) { |
||||||
|
// Error while getting file size or while deleting
|
||||||
|
Log(InfoLogLevel::ERROR_LEVEL, info_log_, |
||||||
|
"Failed to delete %s from trash -- %s", path_in_trash.c_str(), |
||||||
|
s.ToString().c_str()); |
||||||
|
*deleted_bytes = 0; |
||||||
|
} else { |
||||||
|
*deleted_bytes = file_size; |
||||||
|
} |
||||||
|
|
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
void DeleteSchedulerImpl::TEST_WaitForEmptyTrash() { |
||||||
|
MutexLock l(&mu_); |
||||||
|
while (pending_files_ > 0 && !closing_) { |
||||||
|
cv_.Wait(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
DeleteScheduler* NewDeleteScheduler(Env* env, const std::string& trash_dir, |
||||||
|
int64_t rate_bytes_per_sec, |
||||||
|
std::shared_ptr<Logger> info_log, |
||||||
|
bool delete_exisitng_trash, |
||||||
|
Status* status) { |
||||||
|
DeleteScheduler* res = |
||||||
|
new DeleteSchedulerImpl(env, trash_dir, rate_bytes_per_sec, info_log); |
||||||
|
|
||||||
|
Status s; |
||||||
|
if (trash_dir != "") { |
||||||
|
s = env->CreateDirIfMissing(trash_dir); |
||||||
|
if (s.ok() && delete_exisitng_trash) { |
||||||
|
std::vector<std::string> files_in_trash; |
||||||
|
s = env->GetChildren(trash_dir, &files_in_trash); |
||||||
|
if (s.ok()) { |
||||||
|
for (const std::string& trash_file : files_in_trash) { |
||||||
|
if (trash_file == "." || trash_file == "..") { |
||||||
|
continue; |
||||||
|
} |
||||||
|
Status file_delete = res->DeleteFile(trash_dir + "/" + trash_file); |
||||||
|
if (s.ok() && !file_delete.ok()) { |
||||||
|
s = file_delete; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (status) { |
||||||
|
*status = s; |
||||||
|
} |
||||||
|
|
||||||
|
return res; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,81 @@ |
|||||||
|
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <map> |
||||||
|
#include <queue> |
||||||
|
#include <string> |
||||||
|
#include <thread> |
||||||
|
|
||||||
|
#include "port/port.h" |
||||||
|
|
||||||
|
#include "rocksdb/delete_scheduler.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class Env; |
||||||
|
class Logger; |
||||||
|
|
||||||
|
class DeleteSchedulerImpl : public DeleteScheduler { |
||||||
|
public: |
||||||
|
DeleteSchedulerImpl(Env* env, const std::string& trash_dir, |
||||||
|
int64_t rate_bytes_per_sec, |
||||||
|
std::shared_ptr<Logger> info_log); |
||||||
|
|
||||||
|
~DeleteSchedulerImpl(); |
||||||
|
|
||||||
|
// Return delete rate limit in bytes per second
|
||||||
|
int64_t GetRateBytesPerSecond() { return rate_bytes_per_sec_; } |
||||||
|
|
||||||
|
// Move file to trash directory and schedule it's deletion
|
||||||
|
Status DeleteFile(const std::string& fname); |
||||||
|
|
||||||
|
// Wait for all files being deleteing in the background to finish or for
|
||||||
|
// destructor to be called.
|
||||||
|
void TEST_WaitForEmptyTrash(); |
||||||
|
|
||||||
|
// Return a map containing errors that happened in BackgroundEmptyTrash
|
||||||
|
// file_path => error status
|
||||||
|
std::map<std::string, Status> GetBackgroundErrors(); |
||||||
|
|
||||||
|
private: |
||||||
|
Status MoveToTrash(const std::string& file_path, std::string* path_in_trash); |
||||||
|
|
||||||
|
Status DeleteTrashFile(const std::string& path_in_trash, |
||||||
|
uint64_t* deleted_bytes); |
||||||
|
|
||||||
|
void BackgroundEmptyTrash(); |
||||||
|
|
||||||
|
Env* env_; |
||||||
|
// Path to the trash directory
|
||||||
|
std::string trash_dir_; |
||||||
|
// Maximum number of bytes that should be deleted per second
|
||||||
|
int64_t rate_bytes_per_sec_; |
||||||
|
// Mutex to protect queue_, pending_files_, bg_errors_, closing_
|
||||||
|
port::Mutex mu_; |
||||||
|
// Queue of files in trash that need to be deleted
|
||||||
|
std::queue<std::string> queue_; |
||||||
|
// Number of files in trash that are waiting to be deleted
|
||||||
|
int32_t pending_files_; |
||||||
|
// Errors that happened in BackgroundEmptyTrash (file_path => error)
|
||||||
|
std::map<std::string, Status> bg_errors_; |
||||||
|
// Set to true in ~DeleteSchedulerImpl() to force BackgroundEmptyTrash to stop
|
||||||
|
bool closing_; |
||||||
|
// Condition variable signaled in these conditions
|
||||||
|
// - pending_files_ value change from 0 => 1
|
||||||
|
// - pending_files_ value change from 1 => 0
|
||||||
|
// - closing_ value is set to true
|
||||||
|
port::CondVar cv_; |
||||||
|
// Background thread running BackgroundEmptyTrash
|
||||||
|
std::unique_ptr<std::thread> bg_thread_; |
||||||
|
// Mutex to protect threads from file name conflicts
|
||||||
|
port::Mutex file_move_mu_; |
||||||
|
std::shared_ptr<Logger> info_log_; |
||||||
|
static const uint64_t kMicrosInSecond = 1000 * 1000LL; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,439 @@ |
|||||||
|
// Copyright (c) 2015, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#ifndef __STDC_FORMAT_MACROS |
||||||
|
#define __STDC_FORMAT_MACROS |
||||||
|
#endif |
||||||
|
|
||||||
|
#include <inttypes.h> |
||||||
|
#include <atomic> |
||||||
|
#include <thread> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "rocksdb/delete_scheduler.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "util/delete_scheduler_impl.h" |
||||||
|
#include "util/string_util.h" |
||||||
|
#include "util/sync_point.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class DeleteSchedulerTest : public testing::Test { |
||||||
|
public: |
||||||
|
DeleteSchedulerTest() : env_(Env::Default()) { |
||||||
|
dummy_files_dir_ = test::TmpDir(env_) + "/dummy_data_dir"; |
||||||
|
DestroyAndCreateDir(dummy_files_dir_); |
||||||
|
trash_dir_ = test::TmpDir(env_) + "/trash"; |
||||||
|
DestroyAndCreateDir(trash_dir_); |
||||||
|
} |
||||||
|
|
||||||
|
~DeleteSchedulerTest() { |
||||||
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
rocksdb::SyncPoint::GetInstance()->LoadDependency({}); |
||||||
|
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks(); |
||||||
|
DestroyDir(dummy_files_dir_); |
||||||
|
if (delete_scheduler_ != nullptr) { |
||||||
|
delete delete_scheduler_; |
||||||
|
delete_scheduler_ = nullptr; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void WaitForEmptyTrash() { |
||||||
|
reinterpret_cast<DeleteSchedulerImpl*>(delete_scheduler_) |
||||||
|
->TEST_WaitForEmptyTrash(); |
||||||
|
} |
||||||
|
|
||||||
|
void DestroyDir(const std::string& dir) { |
||||||
|
if (env_->FileExists(dir).IsNotFound()) { |
||||||
|
return; |
||||||
|
} |
||||||
|
std::vector<std::string> files_in_dir; |
||||||
|
EXPECT_OK(env_->GetChildren(dir, &files_in_dir)); |
||||||
|
for (auto& file_in_dir : files_in_dir) { |
||||||
|
if (file_in_dir == "." || file_in_dir == "..") { |
||||||
|
continue; |
||||||
|
} |
||||||
|
EXPECT_OK(env_->DeleteFile(dir + "/" + file_in_dir)); |
||||||
|
} |
||||||
|
EXPECT_OK(env_->DeleteDir(dir)); |
||||||
|
} |
||||||
|
|
||||||
|
void DestroyAndCreateDir(const std::string& dir) { |
||||||
|
DestroyDir(dir); |
||||||
|
EXPECT_OK(env_->CreateDir(dir)); |
||||||
|
} |
||||||
|
|
||||||
|
int CountFilesInDir(const std::string& dir) { |
||||||
|
std::vector<std::string> files_in_dir; |
||||||
|
EXPECT_OK(env_->GetChildren(dir, &files_in_dir)); |
||||||
|
// Ignore "." and ".."
|
||||||
|
return static_cast<int>(files_in_dir.size()) - 2; |
||||||
|
} |
||||||
|
|
||||||
|
std::string NewDummyFile(const std::string& file_name, uint64_t size = 1024) { |
||||||
|
std::string file_path = dummy_files_dir_ + "/" + file_name; |
||||||
|
std::unique_ptr<WritableFile> f; |
||||||
|
env_->NewWritableFile(file_path, &f, EnvOptions()); |
||||||
|
std::string data(size, 'A'); |
||||||
|
EXPECT_OK(f->Append(data)); |
||||||
|
EXPECT_OK(f->Close()); |
||||||
|
return file_path; |
||||||
|
} |
||||||
|
|
||||||
|
Env* env_; |
||||||
|
std::string dummy_files_dir_; |
||||||
|
std::string trash_dir_; |
||||||
|
int64_t rate_bytes_per_sec_; |
||||||
|
DeleteScheduler* delete_scheduler_; |
||||||
|
}; |
||||||
|
|
||||||
|
// Test the basic functionality of DeleteScheduler (Rate Limiting).
|
||||||
|
// 1- Create 100 dummy files
|
||||||
|
// 2- Delete the 100 dummy files using DeleteScheduler
|
||||||
|
// 3- Wait for DeleteScheduler to delete all files in trash
|
||||||
|
// 4- Measure time spent in step 2,3 and make sure it matches the expected
|
||||||
|
// time from a rate limited delete
|
||||||
|
// 5- Make sure that all created files were completely deleted
|
||||||
|
TEST_F(DeleteSchedulerTest, BasicRateLimiting) { |
||||||
|
int num_files = 100; // 100 files
|
||||||
|
uint64_t file_size = 1024; // every file is 1 kb
|
||||||
|
std::vector<uint64_t> delete_kbs_per_sec = {512, 200, 100, 50, 25}; |
||||||
|
|
||||||
|
for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) { |
||||||
|
DestroyAndCreateDir(dummy_files_dir_); |
||||||
|
rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024; |
||||||
|
delete_scheduler_ = |
||||||
|
NewDeleteScheduler(env_, trash_dir_, rate_bytes_per_sec_); |
||||||
|
|
||||||
|
// Create 100 dummy files, every file is 1 Kb
|
||||||
|
std::vector<std::string> generated_files; |
||||||
|
uint64_t total_files_size = 0; |
||||||
|
for (int i = 0; i < num_files; i++) { |
||||||
|
std::string file_name = "file" + ToString(i) + ".data"; |
||||||
|
generated_files.push_back(NewDummyFile(file_name, file_size)); |
||||||
|
total_files_size += file_size; |
||||||
|
} |
||||||
|
|
||||||
|
// Delete dummy files and measure time spent to empty trash
|
||||||
|
uint64_t delete_start_time = env_->NowMicros(); |
||||||
|
for (int i = 0; i < num_files; i++) { |
||||||
|
ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[i])); |
||||||
|
} |
||||||
|
ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0); |
||||||
|
|
||||||
|
WaitForEmptyTrash(); |
||||||
|
uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time; |
||||||
|
uint64_t expected_delete_time = |
||||||
|
((total_files_size * 1000000) / rate_bytes_per_sec_); |
||||||
|
ASSERT_GT(time_spent_deleting, expected_delete_time * 0.9); |
||||||
|
ASSERT_LT(time_spent_deleting, expected_delete_time * 1.1); |
||||||
|
printf("Delete time = %" PRIu64 ", Expected delete time = %" PRIu64 |
||||||
|
", Ratio %f\n", |
||||||
|
time_spent_deleting, expected_delete_time, |
||||||
|
static_cast<double>(time_spent_deleting) / expected_delete_time); |
||||||
|
|
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 0); |
||||||
|
auto bg_errors = delete_scheduler_->GetBackgroundErrors(); |
||||||
|
ASSERT_EQ(bg_errors.size(), 0); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Same as the BasicRateLimiting test but delete files in multiple threads.
|
||||||
|
// 1- Create 100 dummy files
|
||||||
|
// 2- Delete the 100 dummy files using DeleteScheduler using 10 threads
|
||||||
|
// 3- Wait for DeleteScheduler to delete all files in queue
|
||||||
|
// 4- Measure time spent in step 2,3 and make sure it matches the expected
|
||||||
|
// time from a rate limited delete
|
||||||
|
// 5- Make sure that all created files were completely deleted
|
||||||
|
TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) { |
||||||
|
int thread_cnt = 10; |
||||||
|
int num_files = 10; // 10 files per thread
|
||||||
|
uint64_t file_size = 1024; // every file is 1 kb
|
||||||
|
std::vector<uint64_t> delete_kbs_per_sec = {512, 200, 100, 50, 25}; |
||||||
|
|
||||||
|
for (size_t t = 0; t < delete_kbs_per_sec.size(); t++) { |
||||||
|
DestroyAndCreateDir(dummy_files_dir_); |
||||||
|
rate_bytes_per_sec_ = delete_kbs_per_sec[t] * 1024; |
||||||
|
delete_scheduler_ = |
||||||
|
NewDeleteScheduler(env_, trash_dir_, rate_bytes_per_sec_); |
||||||
|
|
||||||
|
// Create 100 dummy files, every file is 1 Kb
|
||||||
|
std::vector<std::string> generated_files; |
||||||
|
uint64_t total_files_size = 0; |
||||||
|
for (int i = 0; i < num_files * thread_cnt; i++) { |
||||||
|
std::string file_name = "file" + ToString(i) + ".data"; |
||||||
|
generated_files.push_back(NewDummyFile(file_name, file_size)); |
||||||
|
total_files_size += file_size; |
||||||
|
} |
||||||
|
|
||||||
|
// Delete dummy files using 10 threads and measure time spent to empty trash
|
||||||
|
uint64_t delete_start_time = env_->NowMicros(); |
||||||
|
std::atomic<int> thread_num(0); |
||||||
|
std::vector<std::thread> threads; |
||||||
|
for (int i = 0; i < thread_cnt; i++) { |
||||||
|
threads.emplace_back([&]() { |
||||||
|
int idx = thread_num.fetch_add(1); |
||||||
|
int range_start = idx * num_files; |
||||||
|
int range_end = range_start + num_files; |
||||||
|
for (int j = range_start; j < range_end; j++){ |
||||||
|
ASSERT_OK(delete_scheduler_->DeleteFile(generated_files[j])); |
||||||
|
} |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
for (size_t i = 0; i < threads.size(); i++) { |
||||||
|
threads[i].join(); |
||||||
|
} |
||||||
|
ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0); |
||||||
|
|
||||||
|
WaitForEmptyTrash(); |
||||||
|
uint64_t time_spent_deleting = env_->NowMicros() - delete_start_time; |
||||||
|
uint64_t expected_delete_time = |
||||||
|
((total_files_size * 1000000) / rate_bytes_per_sec_); |
||||||
|
ASSERT_GT(time_spent_deleting, expected_delete_time * 0.9); |
||||||
|
ASSERT_LT(time_spent_deleting, expected_delete_time * 1.1); |
||||||
|
printf("Delete time = %" PRIu64 ", Expected delete time = %" PRIu64 |
||||||
|
", Ratio %f\n", |
||||||
|
time_spent_deleting, expected_delete_time, |
||||||
|
static_cast<double>(time_spent_deleting) / expected_delete_time); |
||||||
|
|
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 0); |
||||||
|
auto bg_errors = delete_scheduler_->GetBackgroundErrors(); |
||||||
|
ASSERT_EQ(bg_errors.size(), 0); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Disable rate limiting by setting rate_bytes_per_sec_ to 0 and make sure
|
||||||
|
// that when DeleteScheduler delete a file it delete it immediately and dont
|
||||||
|
// move it to trash
|
||||||
|
TEST_F(DeleteSchedulerTest, DisableRateLimiting) { |
||||||
|
int bg_delete_file = 0; |
||||||
|
rocksdb::SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"DeleteSchedulerImpl::DeleteTrashFile:DeleteFile", |
||||||
|
[&](void* arg) { bg_delete_file++; }); |
||||||
|
|
||||||
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
delete_scheduler_ = NewDeleteScheduler(env_, "", 0); |
||||||
|
|
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
// Every file we delete will be deleted immediately
|
||||||
|
std::string dummy_file = NewDummyFile("dummy.data"); |
||||||
|
ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file)); |
||||||
|
ASSERT_TRUE(env_->FileExists(dummy_file).IsNotFound()); |
||||||
|
ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0); |
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 0); |
||||||
|
} |
||||||
|
|
||||||
|
ASSERT_EQ(bg_delete_file, 0); |
||||||
|
|
||||||
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
} |
||||||
|
|
||||||
|
// Testing that moving files to trash with the same name is not a problem
|
||||||
|
// 1- Create 10 files with the same name "conflict.data"
|
||||||
|
// 2- Delete the 10 files using DeleteScheduler
|
||||||
|
// 3- Make sure that trash directory contain 10 files ("conflict.data" x 10)
|
||||||
|
// --- Hold DeleteSchedulerImpl::BackgroundEmptyTrash ---
|
||||||
|
// 4- Make sure that files are deleted from trash
|
||||||
|
TEST_F(DeleteSchedulerTest, ConflictNames) { |
||||||
|
rocksdb::SyncPoint::GetInstance()->LoadDependency({ |
||||||
|
{"DeleteSchedulerTest::ConflictNames:1", |
||||||
|
"DeleteSchedulerImpl::BackgroundEmptyTrash"}, |
||||||
|
}); |
||||||
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
rate_bytes_per_sec_ = 1024 * 1024; // 1 Mb/sec
|
||||||
|
delete_scheduler_ = NewDeleteScheduler(env_, trash_dir_, rate_bytes_per_sec_); |
||||||
|
|
||||||
|
// Create "conflict.data" and move it to trash 10 times
|
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
std::string dummy_file = NewDummyFile("conflict.data"); |
||||||
|
ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file)); |
||||||
|
} |
||||||
|
ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0); |
||||||
|
// 10 files ("conflict.data" x 10) in trash
|
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 10); |
||||||
|
|
||||||
|
// Hold BackgroundEmptyTrash
|
||||||
|
TEST_SYNC_POINT("DeleteSchedulerTest::ConflictNames:1"); |
||||||
|
WaitForEmptyTrash(); |
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 0); |
||||||
|
|
||||||
|
auto bg_errors = delete_scheduler_->GetBackgroundErrors(); |
||||||
|
ASSERT_EQ(bg_errors.size(), 0); |
||||||
|
|
||||||
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
} |
||||||
|
|
||||||
|
// 1- Create 10 dummy files
|
||||||
|
// 2- Delete the 10 files using DeleteScheduler (move them to trsah)
|
||||||
|
// 3- Delete the 10 files directly (using env_->DeleteFile)
|
||||||
|
// --- Hold DeleteSchedulerImpl::BackgroundEmptyTrash ---
|
||||||
|
// 4- Make sure that DeleteScheduler failed to delete the 10 files and
|
||||||
|
// reported 10 background errors
|
||||||
|
TEST_F(DeleteSchedulerTest, BackgroundError) { |
||||||
|
rocksdb::SyncPoint::GetInstance()->LoadDependency({ |
||||||
|
{"DeleteSchedulerTest::BackgroundError:1", |
||||||
|
"DeleteSchedulerImpl::BackgroundEmptyTrash"}, |
||||||
|
}); |
||||||
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
rate_bytes_per_sec_ = 1024 * 1024; // 1 Mb/sec
|
||||||
|
delete_scheduler_ = NewDeleteScheduler(env_, trash_dir_, rate_bytes_per_sec_); |
||||||
|
|
||||||
|
// Generate 10 dummy files and move them to trash
|
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
std::string file_name = "data_" + ToString(i) + ".data"; |
||||||
|
ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name))); |
||||||
|
} |
||||||
|
ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0); |
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 10); |
||||||
|
|
||||||
|
// Delete 10 files from trash, this will cause background errors in
|
||||||
|
// BackgroundEmptyTrash since we already deleted the files it was
|
||||||
|
// goind to delete
|
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
std::string file_name = "data_" + ToString(i) + ".data"; |
||||||
|
ASSERT_OK(env_->DeleteFile(trash_dir_ + "/" + file_name)); |
||||||
|
} |
||||||
|
|
||||||
|
// Hold BackgroundEmptyTrash
|
||||||
|
TEST_SYNC_POINT("DeleteSchedulerTest::BackgroundError:1"); |
||||||
|
WaitForEmptyTrash(); |
||||||
|
auto bg_errors = delete_scheduler_->GetBackgroundErrors(); |
||||||
|
ASSERT_EQ(bg_errors.size(), 10); |
||||||
|
|
||||||
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
} |
||||||
|
|
||||||
|
// 1- Create 10 files in trash
|
||||||
|
// 2- Create a DeleteScheduler with delete_exisitng_trash = true
|
||||||
|
// 3- Wait for DeleteScheduler to delete all files in queue
|
||||||
|
// 4- Make sure that all files in trash directory were deleted
|
||||||
|
TEST_F(DeleteSchedulerTest, TrashWithExistingFiles) { |
||||||
|
std::vector<std::string> dummy_files; |
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
std::string file_name = "data_" + ToString(i) + ".data"; |
||||||
|
std::string trash_path = trash_dir_ + "/" + file_name; |
||||||
|
env_->RenameFile(NewDummyFile(file_name), trash_path); |
||||||
|
} |
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 10); |
||||||
|
|
||||||
|
Status s; |
||||||
|
rate_bytes_per_sec_ = 1024 * 1024; // 1 Mb/sec
|
||||||
|
delete_scheduler_ = NewDeleteScheduler(env_, trash_dir_, rate_bytes_per_sec_, |
||||||
|
nullptr, true, &s); |
||||||
|
ASSERT_OK(s); |
||||||
|
|
||||||
|
WaitForEmptyTrash(); |
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 0); |
||||||
|
|
||||||
|
auto bg_errors = delete_scheduler_->GetBackgroundErrors(); |
||||||
|
ASSERT_EQ(bg_errors.size(), 0); |
||||||
|
} |
||||||
|
|
||||||
|
// 1- Create 10 dummy files
|
||||||
|
// 2- Delete 10 dummy files using DeleteScheduler
|
||||||
|
// 3- Wait for DeleteScheduler to delete all files in queue
|
||||||
|
// 4- Make sure all files in trash directory were deleted
|
||||||
|
// 5- Repeat previous steps 5 times
|
||||||
|
TEST_F(DeleteSchedulerTest, StartBGEmptyTrashMultipleTimes) { |
||||||
|
int bg_delete_file = 0; |
||||||
|
rocksdb::SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"DeleteSchedulerImpl::DeleteTrashFile:DeleteFile", |
||||||
|
[&](void* arg) { bg_delete_file++; }); |
||||||
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
rate_bytes_per_sec_ = 1024 * 1024; // 1 MB / sec
|
||||||
|
delete_scheduler_ = NewDeleteScheduler(env_, trash_dir_, rate_bytes_per_sec_); |
||||||
|
|
||||||
|
// Move files to trash, wait for empty trash, start again
|
||||||
|
for (int run = 1; run <= 5; run++) { |
||||||
|
// Generate 10 dummy files and move them to trash
|
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
std::string file_name = "data_" + ToString(i) + ".data"; |
||||||
|
ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name))); |
||||||
|
} |
||||||
|
ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0); |
||||||
|
WaitForEmptyTrash(); |
||||||
|
ASSERT_EQ(bg_delete_file, 10 * run); |
||||||
|
ASSERT_EQ(CountFilesInDir(trash_dir_), 0); |
||||||
|
|
||||||
|
auto bg_errors = delete_scheduler_->GetBackgroundErrors(); |
||||||
|
ASSERT_EQ(bg_errors.size(), 0); |
||||||
|
} |
||||||
|
|
||||||
|
ASSERT_EQ(bg_delete_file, 50); |
||||||
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
} |
||||||
|
|
||||||
|
// 1- Create a DeleteScheduler with very slow rate limit (1 Byte / sec)
|
||||||
|
// 2- Delete 100 files using DeleteScheduler
|
||||||
|
// 3- Delete the DeleteScheduler (call the destructor while queue is not empty)
|
||||||
|
// 4- Make sure that not all files were deleted from trash and that
|
||||||
|
// DeleteScheduler background thread did not delete all files
|
||||||
|
TEST_F(DeleteSchedulerTest, DestructorWithNonEmptyQueue) { |
||||||
|
int bg_delete_file = 0; |
||||||
|
rocksdb::SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"DeleteSchedulerImpl::DeleteTrashFile:DeleteFile", |
||||||
|
[&](void* arg) { bg_delete_file++; }); |
||||||
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
rate_bytes_per_sec_ = 1; // 1 Byte / sec
|
||||||
|
delete_scheduler_ = NewDeleteScheduler(env_, trash_dir_, rate_bytes_per_sec_); |
||||||
|
|
||||||
|
for (int i = 0; i < 100; i++) { |
||||||
|
std::string file_name = "data_" + ToString(i) + ".data"; |
||||||
|
ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name))); |
||||||
|
} |
||||||
|
|
||||||
|
// Deleting 100 files will need >28 hours to delete
|
||||||
|
// we will delete the DeleteScheduler while delete queue is not empty
|
||||||
|
delete delete_scheduler_; |
||||||
|
delete_scheduler_ = nullptr; |
||||||
|
|
||||||
|
ASSERT_LT(bg_delete_file, 100); |
||||||
|
ASSERT_GT(CountFilesInDir(trash_dir_), 0); |
||||||
|
|
||||||
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
} |
||||||
|
|
||||||
|
// 1- Delete the trash directory
|
||||||
|
// 2- Delete 10 files using DeleteScheduler
|
||||||
|
// 3- Make sure that the 10 files were deleted immediately since DeleteScheduler
|
||||||
|
// failed to move them to trash directory
|
||||||
|
TEST_F(DeleteSchedulerTest, MoveToTrashError) { |
||||||
|
int bg_delete_file = 0; |
||||||
|
rocksdb::SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"DeleteSchedulerImpl::DeleteTrashFile:DeleteFile", |
||||||
|
[&](void* arg) { bg_delete_file++; }); |
||||||
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
rate_bytes_per_sec_ = 1024; // 1 Kb / sec
|
||||||
|
delete_scheduler_ = NewDeleteScheduler(env_, trash_dir_, rate_bytes_per_sec_); |
||||||
|
|
||||||
|
// We will delete the trash directory, that mean that DeleteScheduler wont
|
||||||
|
// be able to move files to trash and will delete files them immediately.
|
||||||
|
DestroyDir(trash_dir_); |
||||||
|
for (int i = 0; i < 10; i++) { |
||||||
|
std::string file_name = "data_" + ToString(i) + ".data"; |
||||||
|
ASSERT_OK(delete_scheduler_->DeleteFile(NewDummyFile(file_name))); |
||||||
|
} |
||||||
|
|
||||||
|
ASSERT_EQ(CountFilesInDir(dummy_files_dir_), 0); |
||||||
|
ASSERT_EQ(bg_delete_file, 0); |
||||||
|
|
||||||
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
::testing::InitGoogleTest(&argc, argv); |
||||||
|
return RUN_ALL_TESTS(); |
||||||
|
} |
||||||
Loading…
Reference in new issue