|
|
|
@ -10,6 +10,7 @@ |
|
|
|
|
#include "utilities/backupable_db.h" |
|
|
|
|
#include "db/filename.h" |
|
|
|
|
#include "util/coding.h" |
|
|
|
|
#include "util/crc32c.h" |
|
|
|
|
#include "rocksdb/transaction_log.h" |
|
|
|
|
|
|
|
|
|
#define __STDC_FORMAT_MACROS |
|
|
|
@ -25,11 +26,11 @@ |
|
|
|
|
|
|
|
|
|
namespace rocksdb { |
|
|
|
|
|
|
|
|
|
// -------- BackupEngine class ---------
|
|
|
|
|
class BackupEngine { |
|
|
|
|
// -------- BackupEngineImpl class ---------
|
|
|
|
|
class BackupEngineImpl : public BackupEngine { |
|
|
|
|
public: |
|
|
|
|
BackupEngine(Env* db_env, const BackupableDBOptions& options); |
|
|
|
|
~BackupEngine(); |
|
|
|
|
BackupEngineImpl(Env* db_env, const BackupableDBOptions& options); |
|
|
|
|
~BackupEngineImpl(); |
|
|
|
|
Status CreateNewBackup(DB* db, bool flush_before_backup = false); |
|
|
|
|
Status PurgeOldBackups(uint32_t num_backups_to_keep); |
|
|
|
|
Status DeleteBackup(BackupID backup_id); |
|
|
|
@ -48,12 +49,22 @@ class BackupEngine { |
|
|
|
|
void DeleteBackupsNewerThan(uint64_t sequence_number); |
|
|
|
|
|
|
|
|
|
private: |
|
|
|
|
struct FileInfo { |
|
|
|
|
FileInfo(const std::string& fname, uint64_t sz, uint32_t checksum) |
|
|
|
|
: refs(0), filename(fname), size(sz), checksum_value(checksum) {} |
|
|
|
|
|
|
|
|
|
int refs; |
|
|
|
|
const std::string filename; |
|
|
|
|
const uint64_t size; |
|
|
|
|
uint32_t checksum_value; |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
class BackupMeta { |
|
|
|
|
public: |
|
|
|
|
BackupMeta(const std::string& meta_filename, |
|
|
|
|
std::unordered_map<std::string, int>* file_refs, Env* env) |
|
|
|
|
std::unordered_map<std::string, FileInfo>* file_infos, Env* env) |
|
|
|
|
: timestamp_(0), size_(0), meta_filename_(meta_filename), |
|
|
|
|
file_refs_(file_refs), env_(env) {} |
|
|
|
|
file_infos_(file_infos), env_(env) {} |
|
|
|
|
|
|
|
|
|
~BackupMeta() {} |
|
|
|
|
|
|
|
|
@ -73,7 +84,8 @@ class BackupEngine { |
|
|
|
|
return sequence_number_; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void AddFile(const std::string& filename, uint64_t size); |
|
|
|
|
Status AddFile(const FileInfo& file_info); |
|
|
|
|
|
|
|
|
|
void Delete(); |
|
|
|
|
|
|
|
|
|
bool Empty() { |
|
|
|
@ -96,7 +108,7 @@ class BackupEngine { |
|
|
|
|
std::string const meta_filename_; |
|
|
|
|
// files with relative paths (without "/" prefix!!)
|
|
|
|
|
std::vector<std::string> files_; |
|
|
|
|
std::unordered_map<std::string, int>* file_refs_; |
|
|
|
|
std::unordered_map<std::string, FileInfo>* file_infos_; |
|
|
|
|
Env* env_; |
|
|
|
|
|
|
|
|
|
static const size_t max_backup_meta_file_size_ = 10 * 1024 * 1024; // 10MB
|
|
|
|
@ -141,6 +153,7 @@ class BackupEngine { |
|
|
|
|
Env* dst_env, |
|
|
|
|
bool sync, |
|
|
|
|
uint64_t* size = nullptr, |
|
|
|
|
uint32_t* checksum_value = nullptr, |
|
|
|
|
uint64_t size_limit = 0); |
|
|
|
|
// if size_limit == 0, there is no size limit, copy everything
|
|
|
|
|
Status BackupFile(BackupID backup_id, |
|
|
|
@ -149,15 +162,21 @@ class BackupEngine { |
|
|
|
|
const std::string& src_dir, |
|
|
|
|
const std::string& src_fname, // starts with "/"
|
|
|
|
|
uint64_t size_limit = 0); |
|
|
|
|
|
|
|
|
|
Status CalculateChecksum(const std::string& src, |
|
|
|
|
Env* src_env, |
|
|
|
|
uint64_t size_limit, |
|
|
|
|
uint32_t* checksum_value); |
|
|
|
|
|
|
|
|
|
// Will delete all the files we don't need anymore
|
|
|
|
|
// If full_scan == true, it will do the full scan of files/ directory
|
|
|
|
|
// and delete all the files that are not referenced from backuped_file_refs_
|
|
|
|
|
// and delete all the files that are not referenced from backuped_file_infos__
|
|
|
|
|
void GarbageCollection(bool full_scan); |
|
|
|
|
|
|
|
|
|
// backup state data
|
|
|
|
|
BackupID latest_backup_id_; |
|
|
|
|
std::map<BackupID, BackupMeta> backups_; |
|
|
|
|
std::unordered_map<std::string, int> backuped_file_refs_; |
|
|
|
|
std::unordered_map<std::string, FileInfo> backuped_file_infos_; |
|
|
|
|
std::vector<BackupID> obsolete_backups_; |
|
|
|
|
std::atomic<bool> stop_backup_; |
|
|
|
|
|
|
|
|
@ -169,7 +188,13 @@ class BackupEngine { |
|
|
|
|
static const size_t copy_file_buffer_size_ = 5 * 1024 * 1024LL; // 5MB
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
BackupEngine::BackupEngine(Env* db_env, const BackupableDBOptions& options) |
|
|
|
|
BackupEngine* BackupEngine::NewBackupEngine( |
|
|
|
|
Env* db_env, const BackupableDBOptions& options) { |
|
|
|
|
return new BackupEngineImpl(db_env, options); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
BackupEngineImpl::BackupEngineImpl(Env* db_env, |
|
|
|
|
const BackupableDBOptions& options) |
|
|
|
|
: stop_backup_(false), |
|
|
|
|
options_(options), |
|
|
|
|
db_env_(db_env), |
|
|
|
@ -198,7 +223,7 @@ BackupEngine::BackupEngine(Env* db_env, const BackupableDBOptions& options) |
|
|
|
|
assert(backups_.find(backup_id) == backups_.end()); |
|
|
|
|
backups_.insert(std::make_pair( |
|
|
|
|
backup_id, BackupMeta(GetBackupMetaFile(backup_id), |
|
|
|
|
&backuped_file_refs_, backup_env_))); |
|
|
|
|
&backuped_file_infos_, backup_env_))); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (options_.destroy_old_data) { // Destory old data
|
|
|
|
@ -252,11 +277,9 @@ BackupEngine::BackupEngine(Env* db_env, const BackupableDBOptions& options) |
|
|
|
|
latest_backup_id_); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
BackupEngine::~BackupEngine() { |
|
|
|
|
LogFlush(options_.info_log); |
|
|
|
|
} |
|
|
|
|
BackupEngineImpl::~BackupEngineImpl() { LogFlush(options_.info_log); } |
|
|
|
|
|
|
|
|
|
void BackupEngine::DeleteBackupsNewerThan(uint64_t sequence_number) { |
|
|
|
|
void BackupEngineImpl::DeleteBackupsNewerThan(uint64_t sequence_number) { |
|
|
|
|
for (auto backup : backups_) { |
|
|
|
|
if (backup.second.GetSequenceNumber() > sequence_number) { |
|
|
|
|
Log(options_.info_log, |
|
|
|
@ -276,7 +299,7 @@ void BackupEngine::DeleteBackupsNewerThan(uint64_t sequence_number) { |
|
|
|
|
GarbageCollection(false); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status BackupEngine::CreateNewBackup(DB* db, bool flush_before_backup) { |
|
|
|
|
Status BackupEngineImpl::CreateNewBackup(DB* db, bool flush_before_backup) { |
|
|
|
|
Status s; |
|
|
|
|
std::vector<std::string> live_files; |
|
|
|
|
VectorLogPtr live_wal_files; |
|
|
|
@ -302,7 +325,7 @@ Status BackupEngine::CreateNewBackup(DB* db, bool flush_before_backup) { |
|
|
|
|
assert(backups_.find(new_backup_id) == backups_.end()); |
|
|
|
|
auto ret = backups_.insert(std::make_pair( |
|
|
|
|
new_backup_id, BackupMeta(GetBackupMetaFile(new_backup_id), |
|
|
|
|
&backuped_file_refs_, backup_env_))); |
|
|
|
|
&backuped_file_infos_, backup_env_))); |
|
|
|
|
assert(ret.second == true); |
|
|
|
|
auto& new_backup = ret.first->second; |
|
|
|
|
new_backup.RecordTimestamp(); |
|
|
|
@ -386,7 +409,7 @@ Status BackupEngine::CreateNewBackup(DB* db, bool flush_before_backup) { |
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status BackupEngine::PurgeOldBackups(uint32_t num_backups_to_keep) { |
|
|
|
|
Status BackupEngineImpl::PurgeOldBackups(uint32_t num_backups_to_keep) { |
|
|
|
|
Log(options_.info_log, "Purging old backups, keeping %u", |
|
|
|
|
num_backups_to_keep); |
|
|
|
|
while (num_backups_to_keep < backups_.size()) { |
|
|
|
@ -399,7 +422,7 @@ Status BackupEngine::PurgeOldBackups(uint32_t num_backups_to_keep) { |
|
|
|
|
return Status::OK(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status BackupEngine::DeleteBackup(BackupID backup_id) { |
|
|
|
|
Status BackupEngineImpl::DeleteBackup(BackupID backup_id) { |
|
|
|
|
Log(options_.info_log, "Deleting backup %u", backup_id); |
|
|
|
|
auto backup = backups_.find(backup_id); |
|
|
|
|
if (backup == backups_.end()) { |
|
|
|
@ -412,7 +435,7 @@ Status BackupEngine::DeleteBackup(BackupID backup_id) { |
|
|
|
|
return Status::OK(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void BackupEngine::GetBackupInfo(std::vector<BackupInfo>* backup_info) { |
|
|
|
|
void BackupEngineImpl::GetBackupInfo(std::vector<BackupInfo>* backup_info) { |
|
|
|
|
backup_info->reserve(backups_.size()); |
|
|
|
|
for (auto& backup : backups_) { |
|
|
|
|
if (!backup.second.Empty()) { |
|
|
|
@ -422,9 +445,9 @@ void BackupEngine::GetBackupInfo(std::vector<BackupInfo>* backup_info) { |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status BackupEngine::RestoreDBFromBackup(BackupID backup_id, |
|
|
|
|
const std::string &db_dir, |
|
|
|
|
const std::string &wal_dir) { |
|
|
|
|
Status BackupEngineImpl::RestoreDBFromBackup(BackupID backup_id, |
|
|
|
|
const std::string& db_dir, |
|
|
|
|
const std::string& wal_dir) { |
|
|
|
|
auto backup_itr = backups_.find(backup_id); |
|
|
|
|
if (backup_itr == backups_.end()) { |
|
|
|
|
return Status::NotFound("Backup not found"); |
|
|
|
@ -478,10 +501,19 @@ Status BackupEngine::RestoreDBFromBackup(BackupID backup_id, |
|
|
|
|
"/" + dst; |
|
|
|
|
|
|
|
|
|
Log(options_.info_log, "Restoring %s to %s\n", file.c_str(), dst.c_str()); |
|
|
|
|
s = CopyFile(GetAbsolutePath(file), dst, backup_env_, db_env_, false); |
|
|
|
|
uint32_t checksum_value; |
|
|
|
|
s = CopyFile(GetAbsolutePath(file), dst, backup_env_, db_env_, false, |
|
|
|
|
nullptr /* size */, &checksum_value); |
|
|
|
|
if (!s.ok()) { |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
const auto iter = backuped_file_infos_.find(file); |
|
|
|
|
assert(iter != backuped_file_infos_.end()); |
|
|
|
|
if (iter->second.checksum_value != checksum_value) { |
|
|
|
|
s = Status::Corruption("Checksum check failed"); |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Log(options_.info_log, "Restoring done -- %s\n", s.ToString().c_str()); |
|
|
|
@ -489,7 +521,7 @@ Status BackupEngine::RestoreDBFromBackup(BackupID backup_id, |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// latest backup id is an ASCII representation of latest backup id
|
|
|
|
|
Status BackupEngine::GetLatestBackupFileContents(uint32_t* latest_backup) { |
|
|
|
|
Status BackupEngineImpl::GetLatestBackupFileContents(uint32_t* latest_backup) { |
|
|
|
|
Status s; |
|
|
|
|
unique_ptr<SequentialFile> file; |
|
|
|
|
s = backup_env_->NewSequentialFile(GetLatestBackupFile(), |
|
|
|
@ -519,7 +551,7 @@ Status BackupEngine::GetLatestBackupFileContents(uint32_t* latest_backup) { |
|
|
|
|
// writing 4 bytes to the file is atomic alright, but we should *never*
|
|
|
|
|
// do something like 1. delete file, 2. write new file
|
|
|
|
|
// We write to a tmp file and then atomically rename
|
|
|
|
|
Status BackupEngine::PutLatestBackupFileContents(uint32_t latest_backup) { |
|
|
|
|
Status BackupEngineImpl::PutLatestBackupFileContents(uint32_t latest_backup) { |
|
|
|
|
Status s; |
|
|
|
|
unique_ptr<WritableFile> file; |
|
|
|
|
EnvOptions env_options; |
|
|
|
@ -549,13 +581,11 @@ Status BackupEngine::PutLatestBackupFileContents(uint32_t latest_backup) { |
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status BackupEngine::CopyFile(const std::string& src, |
|
|
|
|
const std::string& dst, |
|
|
|
|
Env* src_env, |
|
|
|
|
Env* dst_env, |
|
|
|
|
bool sync, |
|
|
|
|
uint64_t* size, |
|
|
|
|
uint64_t size_limit) { |
|
|
|
|
Status BackupEngineImpl::CopyFile(const std::string& src, |
|
|
|
|
const std::string& dst, Env* src_env, |
|
|
|
|
Env* dst_env, bool sync, uint64_t* size, |
|
|
|
|
uint32_t* checksum_value, |
|
|
|
|
uint64_t size_limit) { |
|
|
|
|
Status s; |
|
|
|
|
unique_ptr<WritableFile> dst_file; |
|
|
|
|
unique_ptr<SequentialFile> src_file; |
|
|
|
@ -564,6 +594,9 @@ Status BackupEngine::CopyFile(const std::string& src, |
|
|
|
|
if (size != nullptr) { |
|
|
|
|
*size = 0; |
|
|
|
|
} |
|
|
|
|
if (checksum_value != nullptr) { |
|
|
|
|
*checksum_value = 0; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Check if size limit is set. if not, set it to very big number
|
|
|
|
|
if (size_limit == 0) { |
|
|
|
@ -589,12 +622,19 @@ Status BackupEngine::CopyFile(const std::string& src, |
|
|
|
|
copy_file_buffer_size_ : size_limit; |
|
|
|
|
s = src_file->Read(buffer_to_read, &data, buf.get()); |
|
|
|
|
size_limit -= data.size(); |
|
|
|
|
|
|
|
|
|
if (!s.ok()) { |
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (size != nullptr) { |
|
|
|
|
*size += data.size(); |
|
|
|
|
} |
|
|
|
|
if (s.ok()) { |
|
|
|
|
s = dst_file->Append(data); |
|
|
|
|
if (checksum_value != nullptr) { |
|
|
|
|
*checksum_value = crc32c::Extend(*checksum_value, data.data(), |
|
|
|
|
data.size()); |
|
|
|
|
} |
|
|
|
|
s = dst_file->Append(data); |
|
|
|
|
} while (s.ok() && data.size() > 0 && size_limit > 0); |
|
|
|
|
|
|
|
|
|
if (s.ok() && sync) { |
|
|
|
@ -605,12 +645,10 @@ Status BackupEngine::CopyFile(const std::string& src, |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// src_fname will always start with "/"
|
|
|
|
|
Status BackupEngine::BackupFile(BackupID backup_id, |
|
|
|
|
BackupMeta* backup, |
|
|
|
|
bool shared, |
|
|
|
|
const std::string& src_dir, |
|
|
|
|
const std::string& src_fname, |
|
|
|
|
uint64_t size_limit) { |
|
|
|
|
Status BackupEngineImpl::BackupFile(BackupID backup_id, BackupMeta* backup, |
|
|
|
|
bool shared, const std::string& src_dir, |
|
|
|
|
const std::string& src_fname, |
|
|
|
|
uint64_t size_limit) { |
|
|
|
|
|
|
|
|
|
assert(src_fname.size() > 0 && src_fname[0] == '/'); |
|
|
|
|
std::string dst_relative = src_fname.substr(1); |
|
|
|
@ -629,9 +667,15 @@ Status BackupEngine::BackupFile(BackupID backup_id, |
|
|
|
|
|
|
|
|
|
// if it's shared, we also need to check if it exists -- if it does,
|
|
|
|
|
// no need to copy it again
|
|
|
|
|
uint32_t checksum_value = 0; |
|
|
|
|
if (shared && backup_env_->FileExists(dst_path)) { |
|
|
|
|
backup_env_->GetFileSize(dst_path, &size); // Ignore error
|
|
|
|
|
Log(options_.info_log, "%s already present", src_fname.c_str()); |
|
|
|
|
Log(options_.info_log, "%s already present, calculate checksum", |
|
|
|
|
src_fname.c_str()); |
|
|
|
|
s = CalculateChecksum(src_dir + src_fname, |
|
|
|
|
db_env_, |
|
|
|
|
size_limit, |
|
|
|
|
&checksum_value); |
|
|
|
|
} else { |
|
|
|
|
Log(options_.info_log, "Copying %s", src_fname.c_str()); |
|
|
|
|
s = CopyFile(src_dir + src_fname, |
|
|
|
@ -640,22 +684,62 @@ Status BackupEngine::BackupFile(BackupID backup_id, |
|
|
|
|
backup_env_, |
|
|
|
|
options_.sync, |
|
|
|
|
&size, |
|
|
|
|
&checksum_value, |
|
|
|
|
size_limit); |
|
|
|
|
if (s.ok() && shared) { |
|
|
|
|
s = backup_env_->RenameFile(dst_path_tmp, dst_path); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if (s.ok()) { |
|
|
|
|
backup->AddFile(dst_relative, size); |
|
|
|
|
s = backup->AddFile(FileInfo(dst_relative, size, checksum_value)); |
|
|
|
|
} |
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void BackupEngine::GarbageCollection(bool full_scan) { |
|
|
|
|
Status BackupEngineImpl::CalculateChecksum(const std::string& src, Env* src_env, |
|
|
|
|
uint64_t size_limit, |
|
|
|
|
uint32_t* checksum_value) { |
|
|
|
|
*checksum_value = 0; |
|
|
|
|
if (size_limit == 0) { |
|
|
|
|
size_limit = std::numeric_limits<uint64_t>::max(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
EnvOptions env_options; |
|
|
|
|
env_options.use_mmap_writes = false; |
|
|
|
|
|
|
|
|
|
std::unique_ptr<SequentialFile> src_file; |
|
|
|
|
Status s = src_env->NewSequentialFile(src, &src_file, env_options); |
|
|
|
|
if (!s.ok()) { |
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
std::unique_ptr<char[]> buf(new char[copy_file_buffer_size_]); |
|
|
|
|
Slice data; |
|
|
|
|
|
|
|
|
|
do { |
|
|
|
|
if (stop_backup_.load(std::memory_order_acquire)) { |
|
|
|
|
return Status::Incomplete("Backup stopped"); |
|
|
|
|
} |
|
|
|
|
size_t buffer_to_read = (copy_file_buffer_size_ < size_limit) ? |
|
|
|
|
copy_file_buffer_size_ : size_limit; |
|
|
|
|
s = src_file->Read(buffer_to_read, &data, buf.get()); |
|
|
|
|
|
|
|
|
|
if (!s.ok()) { |
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
size_limit -= data.size(); |
|
|
|
|
*checksum_value = crc32c::Extend(*checksum_value, data.data(), data.size()); |
|
|
|
|
} while (data.size() > 0 && size_limit > 0); |
|
|
|
|
|
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void BackupEngineImpl::GarbageCollection(bool full_scan) { |
|
|
|
|
Log(options_.info_log, "Starting garbage collection"); |
|
|
|
|
std::vector<std::string> to_delete; |
|
|
|
|
for (auto& itr : backuped_file_refs_) { |
|
|
|
|
if (itr.second == 0) { |
|
|
|
|
for (auto& itr : backuped_file_infos_) { |
|
|
|
|
if (itr.second.refs == 0) { |
|
|
|
|
Status s = backup_env_->DeleteFile(GetAbsolutePath(itr.first)); |
|
|
|
|
Log(options_.info_log, "Deleting %s -- %s", itr.first.c_str(), |
|
|
|
|
s.ToString().c_str()); |
|
|
|
@ -663,7 +747,7 @@ void BackupEngine::GarbageCollection(bool full_scan) { |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
for (auto& td : to_delete) { |
|
|
|
|
backuped_file_refs_.erase(td); |
|
|
|
|
backuped_file_infos_.erase(td); |
|
|
|
|
} |
|
|
|
|
if (!full_scan) { |
|
|
|
|
// take care of private dirs -- if full_scan == true, then full_scan will
|
|
|
|
@ -686,7 +770,7 @@ void BackupEngine::GarbageCollection(bool full_scan) { |
|
|
|
|
for (auto& child : shared_children) { |
|
|
|
|
std::string rel_fname = GetSharedFileRel(child); |
|
|
|
|
// if it's not refcounted, delete it
|
|
|
|
|
if (backuped_file_refs_.find(rel_fname) == backuped_file_refs_.end()) { |
|
|
|
|
if (backuped_file_infos_.find(rel_fname) == backuped_file_infos_.end()) { |
|
|
|
|
// this might be a directory, but DeleteFile will just fail in that
|
|
|
|
|
// case, so we're good
|
|
|
|
|
Status s = backup_env_->DeleteFile(GetAbsolutePath(rel_fname)); |
|
|
|
@ -731,23 +815,34 @@ void BackupEngine::GarbageCollection(bool full_scan) { |
|
|
|
|
|
|
|
|
|
// ------- BackupMeta class --------
|
|
|
|
|
|
|
|
|
|
void BackupEngine::BackupMeta::AddFile(const std::string& filename, |
|
|
|
|
uint64_t size) { |
|
|
|
|
size_ += size; |
|
|
|
|
files_.push_back(filename); |
|
|
|
|
auto itr = file_refs_->find(filename); |
|
|
|
|
if (itr == file_refs_->end()) { |
|
|
|
|
file_refs_->insert(std::make_pair(filename, 1)); |
|
|
|
|
Status BackupEngineImpl::BackupMeta::AddFile(const FileInfo& file_info) { |
|
|
|
|
size_ += file_info.size; |
|
|
|
|
files_.push_back(file_info.filename); |
|
|
|
|
|
|
|
|
|
auto itr = file_infos_->find(file_info.filename); |
|
|
|
|
if (itr == file_infos_->end()) { |
|
|
|
|
auto ret = file_infos_->insert({file_info.filename, file_info}); |
|
|
|
|
if (ret.second) { |
|
|
|
|
ret.first->second.refs = 1; |
|
|
|
|
} else { |
|
|
|
|
// if this happens, something is seriously wrong
|
|
|
|
|
return Status::Corruption("In memory metadata insertion error"); |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
++itr->second; // increase refcount if already present
|
|
|
|
|
if (itr->second.checksum_value != file_info.checksum_value) { |
|
|
|
|
return Status::Corruption("Checksum mismatch for existing backup file"); |
|
|
|
|
} |
|
|
|
|
++itr->second.refs; // increase refcount if already present
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return Status::OK(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
void BackupEngine::BackupMeta::Delete() { |
|
|
|
|
for (auto& file : files_) { |
|
|
|
|
auto itr = file_refs_->find(file); |
|
|
|
|
assert(itr != file_refs_->end()); |
|
|
|
|
--(itr->second); // decrease refcount
|
|
|
|
|
void BackupEngineImpl::BackupMeta::Delete() { |
|
|
|
|
for (const auto& file : files_) { |
|
|
|
|
auto itr = file_infos_->find(file); |
|
|
|
|
assert(itr != file_infos_->end()); |
|
|
|
|
--(itr->second.refs); // decrease refcount
|
|
|
|
|
} |
|
|
|
|
files_.clear(); |
|
|
|
|
// delete meta file
|
|
|
|
@ -759,11 +854,12 @@ void BackupEngine::BackupMeta::Delete() { |
|
|
|
|
// <timestamp>
|
|
|
|
|
// <seq number>
|
|
|
|
|
// <number of files>
|
|
|
|
|
// <file1>
|
|
|
|
|
// <file2>
|
|
|
|
|
// <file1> <crc32(literal string)> <crc32_value>
|
|
|
|
|
// <file2> <crc32(literal string)> <crc32_value>
|
|
|
|
|
// ...
|
|
|
|
|
// TODO: maybe add checksum?
|
|
|
|
|
Status BackupEngine::BackupMeta::LoadFromFile(const std::string& backup_dir) { |
|
|
|
|
Status BackupEngineImpl::BackupMeta::LoadFromFile( |
|
|
|
|
const std::string& backup_dir) { |
|
|
|
|
assert(Empty()); |
|
|
|
|
Status s; |
|
|
|
|
unique_ptr<SequentialFile> backup_meta_file; |
|
|
|
@ -790,25 +886,47 @@ Status BackupEngine::BackupMeta::LoadFromFile(const std::string& backup_dir) { |
|
|
|
|
sscanf(data.data(), "%u%n", &num_files, &bytes_read); |
|
|
|
|
data.remove_prefix(bytes_read + 1); // +1 for '\n'
|
|
|
|
|
|
|
|
|
|
std::vector<std::pair<std::string, uint64_t>> files; |
|
|
|
|
std::vector<FileInfo> files; |
|
|
|
|
|
|
|
|
|
for (uint32_t i = 0; s.ok() && i < num_files; ++i) { |
|
|
|
|
std::string filename = GetSliceUntil(&data, '\n').ToString(); |
|
|
|
|
auto line = GetSliceUntil(&data, '\n'); |
|
|
|
|
std::string filename = GetSliceUntil(&line, ' ').ToString(); |
|
|
|
|
|
|
|
|
|
uint64_t size; |
|
|
|
|
s = env_->GetFileSize(backup_dir + "/" + filename, &size); |
|
|
|
|
files.push_back(std::make_pair(filename, size)); |
|
|
|
|
|
|
|
|
|
if (line.empty()) { |
|
|
|
|
return Status::Corruption("File checksum is missing"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
uint32_t checksum_value = 0; |
|
|
|
|
if (line.starts_with("crc32 ")) { |
|
|
|
|
line.remove_prefix(6); |
|
|
|
|
sscanf(line.data(), "%u", &checksum_value); |
|
|
|
|
if (memcmp(line.data(), std::to_string(checksum_value).c_str(), |
|
|
|
|
line.size() - 1) != 0) { |
|
|
|
|
return Status::Corruption("Invalid checksum value"); |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
return Status::Corruption("Unknown checksum type"); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
files.emplace_back(filename, size, checksum_value); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if (s.ok()) { |
|
|
|
|
for (auto file : files) { |
|
|
|
|
AddFile(file.first, file.second); |
|
|
|
|
for (const auto& file_info : files) { |
|
|
|
|
s = AddFile(file_info); |
|
|
|
|
if (!s.ok()) { |
|
|
|
|
break; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status BackupEngine::BackupMeta::StoreToFile(bool sync) { |
|
|
|
|
Status BackupEngineImpl::BackupMeta::StoreToFile(bool sync) { |
|
|
|
|
Status s; |
|
|
|
|
unique_ptr<WritableFile> backup_meta_file; |
|
|
|
|
EnvOptions env_options; |
|
|
|
@ -825,8 +943,13 @@ Status BackupEngine::BackupMeta::StoreToFile(bool sync) { |
|
|
|
|
len += snprintf(buf.get() + len, buf_size - len, "%" PRIu64 "\n", |
|
|
|
|
sequence_number_); |
|
|
|
|
len += snprintf(buf.get() + len, buf_size - len, "%zu\n", files_.size()); |
|
|
|
|
for (size_t i = 0; i < files_.size(); ++i) { |
|
|
|
|
len += snprintf(buf.get() + len, buf_size - len, "%s\n", files_[i].c_str()); |
|
|
|
|
for (const auto& file : files_) { |
|
|
|
|
const auto& iter = file_infos_->find(file); |
|
|
|
|
|
|
|
|
|
assert(iter != file_infos_->end()); |
|
|
|
|
// use crc32 for now, switch to something else if needed
|
|
|
|
|
len += snprintf(buf.get() + len, buf_size - len, "%s crc32 %u\n", |
|
|
|
|
file.c_str(), iter->second.checksum_value); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
s = backup_meta_file->Append(Slice(buf.get(), (size_t)len)); |
|
|
|
@ -845,7 +968,8 @@ Status BackupEngine::BackupMeta::StoreToFile(bool sync) { |
|
|
|
|
// --- BackupableDB methods --------
|
|
|
|
|
|
|
|
|
|
BackupableDB::BackupableDB(DB* db, const BackupableDBOptions& options) |
|
|
|
|
: StackableDB(db), backup_engine_(new BackupEngine(db->GetEnv(), options)) { |
|
|
|
|
: StackableDB(db), |
|
|
|
|
backup_engine_(new BackupEngineImpl(db->GetEnv(), options)) { |
|
|
|
|
if (options.share_table_files) { |
|
|
|
|
backup_engine_->DeleteBackupsNewerThan(GetLatestSequenceNumber()); |
|
|
|
|
} |
|
|
|
@ -879,7 +1003,7 @@ void BackupableDB::StopBackup() { |
|
|
|
|
|
|
|
|
|
RestoreBackupableDB::RestoreBackupableDB(Env* db_env, |
|
|
|
|
const BackupableDBOptions& options) |
|
|
|
|
: backup_engine_(new BackupEngine(db_env, options)) {} |
|
|
|
|
: backup_engine_(new BackupEngineImpl(db_env, options)) {} |
|
|
|
|
|
|
|
|
|
RestoreBackupableDB::~RestoreBackupableDB() { |
|
|
|
|
delete backup_engine_; |
|
|
|
|