Deprecate obsolete "backupable db" from public APIs (#8274)
Summary: An early design of BackupEngine used stackable DB, so I guess a DB had to opt-in to being backupable. Unfortunately the naming of that obsolete design still infects our public API and implementation. This change fixes the public API, with a deprecated backward-compatibility header. `BackupableDBOptions` is renamed to `BackupEngineOptions` (copy-replace in the public header) and backup_engine.h replaces backupable_db.h (present for backward compatibility). The only other change in backupable_db.h -> backup_engine.h is cleaning up headers. Later changes will fix the internal implementation. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8274 Test Plan: The internal implementation of BackupEngine uses the name BackupEngineOptions, while the unit tests use the old name BackupableDBOptions. This gives me confidence that both still work. Reviewed By: mrambacher Differential Revision: D28259471 Pulled By: pdillinger fbshipit-source-id: a25dbe327b9772143488e7bb0ec7139ee42d0613main
Peter Dillinger
4 years ago
committed by
Facebook GitHub Bot
parent
a4919d6b62
commit
c26b75baa5
@ -0,0 +1,606 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <cstdint> |
||||||
|
#include <functional> |
||||||
|
#include <map> |
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
// The default DB file checksum function name.
|
||||||
|
constexpr char kDbFileChecksumFuncName[] = "FileChecksumCrc32c"; |
||||||
|
// The default BackupEngine file checksum function name.
|
||||||
|
constexpr char kBackupFileChecksumFuncName[] = "crc32c"; |
||||||
|
|
||||||
|
struct BackupEngineOptions { |
||||||
|
// Where to keep the backup files. Has to be different than dbname_
|
||||||
|
// Best to set this to dbname_ + "/backups"
|
||||||
|
// Required
|
||||||
|
std::string backup_dir; |
||||||
|
|
||||||
|
// Backup Env object. It will be used for backup file I/O. If it's
|
||||||
|
// nullptr, backups will be written out using DBs Env. If it's
|
||||||
|
// non-nullptr, backup's I/O will be performed using this object.
|
||||||
|
// If you want to have backups on HDFS, use HDFS Env here!
|
||||||
|
// Default: nullptr
|
||||||
|
Env* backup_env; |
||||||
|
|
||||||
|
// share_table_files supports table and blob files.
|
||||||
|
//
|
||||||
|
// If share_table_files == true, the backup directory will share table and
|
||||||
|
// blob files among backups, to save space among backups of the same DB and to
|
||||||
|
// enable incremental backups by only copying new files.
|
||||||
|
// If share_table_files == false, each backup will be on its own and will not
|
||||||
|
// share any data with other backups.
|
||||||
|
//
|
||||||
|
// default: true
|
||||||
|
bool share_table_files; |
||||||
|
|
||||||
|
// Backup info and error messages will be written to info_log
|
||||||
|
// if non-nullptr.
|
||||||
|
// Default: nullptr
|
||||||
|
Logger* info_log; |
||||||
|
|
||||||
|
// If sync == true, we can guarantee you'll get consistent backup even
|
||||||
|
// on a machine crash/reboot. Backup process is slower with sync enabled.
|
||||||
|
// If sync == false, we don't guarantee anything on machine reboot. However,
|
||||||
|
// chances are some of the backups are consistent.
|
||||||
|
// Default: true
|
||||||
|
bool sync; |
||||||
|
|
||||||
|
// If true, it will delete whatever backups there are already
|
||||||
|
// Default: false
|
||||||
|
bool destroy_old_data; |
||||||
|
|
||||||
|
// If false, we won't backup log files. This option can be useful for backing
|
||||||
|
// up in-memory databases where log file are persisted, but table files are in
|
||||||
|
// memory.
|
||||||
|
// Default: true
|
||||||
|
bool backup_log_files; |
||||||
|
|
||||||
|
// Max bytes that can be transferred in a second during backup.
|
||||||
|
// If 0, go as fast as you can
|
||||||
|
// Default: 0
|
||||||
|
uint64_t backup_rate_limit; |
||||||
|
|
||||||
|
// Backup rate limiter. Used to control transfer speed for backup. If this is
|
||||||
|
// not null, backup_rate_limit is ignored.
|
||||||
|
// Default: nullptr
|
||||||
|
std::shared_ptr<RateLimiter> backup_rate_limiter{nullptr}; |
||||||
|
|
||||||
|
// Max bytes that can be transferred in a second during restore.
|
||||||
|
// If 0, go as fast as you can
|
||||||
|
// Default: 0
|
||||||
|
uint64_t restore_rate_limit; |
||||||
|
|
||||||
|
// Restore rate limiter. Used to control transfer speed during restore. If
|
||||||
|
// this is not null, restore_rate_limit is ignored.
|
||||||
|
// Default: nullptr
|
||||||
|
std::shared_ptr<RateLimiter> restore_rate_limiter{nullptr}; |
||||||
|
|
||||||
|
// share_files_with_checksum supports table and blob files.
|
||||||
|
//
|
||||||
|
// Only used if share_table_files is set to true. Setting to false is
|
||||||
|
// DEPRECATED and potentially dangerous because in that case BackupEngine
|
||||||
|
// can lose data if backing up databases with distinct or divergent
|
||||||
|
// history, for example if restoring from a backup other than the latest,
|
||||||
|
// writing to the DB, and creating another backup. Setting to true (default)
|
||||||
|
// prevents these issues by ensuring that different table files (SSTs) and
|
||||||
|
// blob files with the same number are treated as distinct. See
|
||||||
|
// share_files_with_checksum_naming and ShareFilesNaming.
|
||||||
|
//
|
||||||
|
// Default: true
|
||||||
|
bool share_files_with_checksum; |
||||||
|
|
||||||
|
// Up to this many background threads will copy files for CreateNewBackup()
|
||||||
|
// and RestoreDBFromBackup()
|
||||||
|
// Default: 1
|
||||||
|
int max_background_operations; |
||||||
|
|
||||||
|
// During backup user can get callback every time next
|
||||||
|
// callback_trigger_interval_size bytes being copied.
|
||||||
|
// Default: 4194304
|
||||||
|
uint64_t callback_trigger_interval_size; |
||||||
|
|
||||||
|
// For BackupEngineReadOnly, Open() will open at most this many of the
|
||||||
|
// latest non-corrupted backups.
|
||||||
|
//
|
||||||
|
// Note: this setting is ignored (behaves like INT_MAX) for any kind of
|
||||||
|
// writable BackupEngine because it would inhibit accounting for shared
|
||||||
|
// files for proper backup deletion, including purging any incompletely
|
||||||
|
// created backups on creation of a new backup.
|
||||||
|
//
|
||||||
|
// Default: INT_MAX
|
||||||
|
int max_valid_backups_to_open; |
||||||
|
|
||||||
|
// ShareFilesNaming describes possible naming schemes for backup
|
||||||
|
// table and blob file names when they are stored in the
|
||||||
|
// shared_checksum directory (i.e., both share_table_files and
|
||||||
|
// share_files_with_checksum are true).
|
||||||
|
enum ShareFilesNaming : uint32_t { |
||||||
|
// Backup blob filenames are <file_number>_<crc32c>_<file_size>.blob and
|
||||||
|
// backup SST filenames are <file_number>_<crc32c>_<file_size>.sst
|
||||||
|
// where <crc32c> is an unsigned decimal integer. This is the
|
||||||
|
// original/legacy naming scheme for share_files_with_checksum,
|
||||||
|
// with two problems:
|
||||||
|
// * At massive scale, collisions on this triple with different file
|
||||||
|
// contents is plausible.
|
||||||
|
// * Determining the name to use requires computing the checksum,
|
||||||
|
// so generally requires reading the whole file even if the file
|
||||||
|
// is already backed up.
|
||||||
|
//
|
||||||
|
// ** ONLY RECOMMENDED FOR PRESERVING OLD BEHAVIOR **
|
||||||
|
kLegacyCrc32cAndFileSize = 1U, |
||||||
|
|
||||||
|
// Backup SST filenames are <file_number>_s<db_session_id>.sst. This
|
||||||
|
// pair of values should be very strongly unique for a given SST file
|
||||||
|
// and easily determined before computing a checksum. The 's' indicates
|
||||||
|
// the value is a DB session id, not a checksum.
|
||||||
|
//
|
||||||
|
// Exceptions:
|
||||||
|
// * For blob files, kLegacyCrc32cAndFileSize is used as currently
|
||||||
|
// db_session_id is not supported by the blob file format.
|
||||||
|
// * For old SST files without a DB session id, kLegacyCrc32cAndFileSize
|
||||||
|
// will be used instead, matching the names assigned by RocksDB versions
|
||||||
|
// not supporting the newer naming scheme.
|
||||||
|
// * See also flags below.
|
||||||
|
kUseDbSessionId = 2U, |
||||||
|
|
||||||
|
kMaskNoNamingFlags = 0xffffU, |
||||||
|
|
||||||
|
// If not already part of the naming scheme, insert
|
||||||
|
// _<file_size>
|
||||||
|
// before .sst and .blob in the name. In case of user code actually parsing
|
||||||
|
// the last _<whatever> before the .sst and .blob as the file size, this
|
||||||
|
// preserves that feature of kLegacyCrc32cAndFileSize. In other words, this
|
||||||
|
// option makes official that unofficial feature of the backup metadata.
|
||||||
|
//
|
||||||
|
// We do not consider SST and blob file sizes to have sufficient entropy to
|
||||||
|
// contribute significantly to naming uniqueness.
|
||||||
|
kFlagIncludeFileSize = 1U << 31, |
||||||
|
|
||||||
|
kMaskNamingFlags = ~kMaskNoNamingFlags, |
||||||
|
}; |
||||||
|
|
||||||
|
// Naming option for share_files_with_checksum table and blob files. See
|
||||||
|
// ShareFilesNaming for details.
|
||||||
|
//
|
||||||
|
// Modifying this option cannot introduce a downgrade compatibility issue
|
||||||
|
// because RocksDB can read, restore, and delete backups using different file
|
||||||
|
// names, and it's OK for a backup directory to use a mixture of table and
|
||||||
|
// blob files naming schemes.
|
||||||
|
//
|
||||||
|
// However, modifying this option and saving more backups to the same
|
||||||
|
// directory can lead to the same file getting saved again to that
|
||||||
|
// directory, under the new shared name in addition to the old shared
|
||||||
|
// name.
|
||||||
|
//
|
||||||
|
// Default: kUseDbSessionId | kFlagIncludeFileSize
|
||||||
|
//
|
||||||
|
// Note: This option comes into effect only if both share_files_with_checksum
|
||||||
|
// and share_table_files are true.
|
||||||
|
ShareFilesNaming share_files_with_checksum_naming; |
||||||
|
|
||||||
|
void Dump(Logger* logger) const; |
||||||
|
|
||||||
|
explicit BackupEngineOptions( |
||||||
|
const std::string& _backup_dir, Env* _backup_env = nullptr, |
||||||
|
bool _share_table_files = true, Logger* _info_log = nullptr, |
||||||
|
bool _sync = true, bool _destroy_old_data = false, |
||||||
|
bool _backup_log_files = true, uint64_t _backup_rate_limit = 0, |
||||||
|
uint64_t _restore_rate_limit = 0, int _max_background_operations = 1, |
||||||
|
uint64_t _callback_trigger_interval_size = 4 * 1024 * 1024, |
||||||
|
int _max_valid_backups_to_open = INT_MAX, |
||||||
|
ShareFilesNaming _share_files_with_checksum_naming = |
||||||
|
static_cast<ShareFilesNaming>(kUseDbSessionId | kFlagIncludeFileSize)) |
||||||
|
: backup_dir(_backup_dir), |
||||||
|
backup_env(_backup_env), |
||||||
|
share_table_files(_share_table_files), |
||||||
|
info_log(_info_log), |
||||||
|
sync(_sync), |
||||||
|
destroy_old_data(_destroy_old_data), |
||||||
|
backup_log_files(_backup_log_files), |
||||||
|
backup_rate_limit(_backup_rate_limit), |
||||||
|
restore_rate_limit(_restore_rate_limit), |
||||||
|
share_files_with_checksum(true), |
||||||
|
max_background_operations(_max_background_operations), |
||||||
|
callback_trigger_interval_size(_callback_trigger_interval_size), |
||||||
|
max_valid_backups_to_open(_max_valid_backups_to_open), |
||||||
|
share_files_with_checksum_naming(_share_files_with_checksum_naming) { |
||||||
|
assert(share_table_files || !share_files_with_checksum); |
||||||
|
assert((share_files_with_checksum_naming & kMaskNoNamingFlags) != 0); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
inline BackupEngineOptions::ShareFilesNaming operator&( |
||||||
|
BackupEngineOptions::ShareFilesNaming lhs, |
||||||
|
BackupEngineOptions::ShareFilesNaming rhs) { |
||||||
|
uint32_t l = static_cast<uint32_t>(lhs); |
||||||
|
uint32_t r = static_cast<uint32_t>(rhs); |
||||||
|
assert(r == BackupEngineOptions::kMaskNoNamingFlags || |
||||||
|
(r & BackupEngineOptions::kMaskNoNamingFlags) == 0); |
||||||
|
return static_cast<BackupEngineOptions::ShareFilesNaming>(l & r); |
||||||
|
} |
||||||
|
|
||||||
|
inline BackupEngineOptions::ShareFilesNaming operator|( |
||||||
|
BackupEngineOptions::ShareFilesNaming lhs, |
||||||
|
BackupEngineOptions::ShareFilesNaming rhs) { |
||||||
|
uint32_t l = static_cast<uint32_t>(lhs); |
||||||
|
uint32_t r = static_cast<uint32_t>(rhs); |
||||||
|
assert((r & BackupEngineOptions::kMaskNoNamingFlags) == 0); |
||||||
|
return static_cast<BackupEngineOptions::ShareFilesNaming>(l | r); |
||||||
|
} |
||||||
|
|
||||||
|
struct CreateBackupOptions { |
||||||
|
// Flush will always trigger if 2PC is enabled.
|
||||||
|
// If write-ahead logs are disabled, set flush_before_backup=true to
|
||||||
|
// avoid losing unflushed key/value pairs from the memtable.
|
||||||
|
bool flush_before_backup = false; |
||||||
|
|
||||||
|
// Callback for reporting progress, based on callback_trigger_interval_size.
|
||||||
|
std::function<void()> progress_callback = []() {}; |
||||||
|
|
||||||
|
// If false, background_thread_cpu_priority is ignored.
|
||||||
|
// Otherwise, the cpu priority can be decreased,
|
||||||
|
// if you try to increase the priority, the priority will not change.
|
||||||
|
// The initial priority of the threads is CpuPriority::kNormal,
|
||||||
|
// so you can decrease to priorities lower than kNormal.
|
||||||
|
bool decrease_background_thread_cpu_priority = false; |
||||||
|
CpuPriority background_thread_cpu_priority = CpuPriority::kNormal; |
||||||
|
}; |
||||||
|
|
||||||
|
struct RestoreOptions { |
||||||
|
// If true, restore won't overwrite the existing log files in wal_dir. It will
|
||||||
|
// also move all log files from archive directory to wal_dir. Use this option
|
||||||
|
// in combination with BackupEngineOptions::backup_log_files = false for
|
||||||
|
// persisting in-memory databases.
|
||||||
|
// Default: false
|
||||||
|
bool keep_log_files; |
||||||
|
|
||||||
|
explicit RestoreOptions(bool _keep_log_files = false) |
||||||
|
: keep_log_files(_keep_log_files) {} |
||||||
|
}; |
||||||
|
|
||||||
|
struct BackupFileInfo { |
||||||
|
// File name and path relative to the backup_dir directory.
|
||||||
|
std::string relative_filename; |
||||||
|
|
||||||
|
// Size of the file in bytes, not including filesystem overheads.
|
||||||
|
uint64_t size; |
||||||
|
}; |
||||||
|
|
||||||
|
typedef uint32_t BackupID; |
||||||
|
|
||||||
|
struct BackupInfo { |
||||||
|
BackupID backup_id = 0U; |
||||||
|
// Creation time, according to GetCurrentTime
|
||||||
|
int64_t timestamp = 0; |
||||||
|
|
||||||
|
// Total size in bytes (based on file payloads, not including filesystem
|
||||||
|
// overheads or backup meta file)
|
||||||
|
uint64_t size = 0U; |
||||||
|
|
||||||
|
// Number of backed up files, some of which might be shared with other
|
||||||
|
// backups. Does not include backup meta file.
|
||||||
|
uint32_t number_files = 0U; |
||||||
|
|
||||||
|
// Backup API user metadata
|
||||||
|
std::string app_metadata; |
||||||
|
|
||||||
|
// Backup file details, if requested with include_file_details=true
|
||||||
|
std::vector<BackupFileInfo> file_details; |
||||||
|
|
||||||
|
// DB "name" (a directory in the backup_env) for opening this backup as a
|
||||||
|
// read-only DB. This should also be used as the DBOptions::wal_dir, such
|
||||||
|
// as by default setting wal_dir="". See also env_for_open.
|
||||||
|
// This field is only set if include_file_details=true
|
||||||
|
std::string name_for_open; |
||||||
|
|
||||||
|
// An Env(+FileSystem) for opening this backup as a read-only DB, with
|
||||||
|
// DB::OpenForReadOnly or similar. This field is only set if
|
||||||
|
// include_file_details=true. (The FileSystem in this Env takes care
|
||||||
|
// of making shared backup files openable from the `name_for_open` DB
|
||||||
|
// directory.) See also name_for_open.
|
||||||
|
//
|
||||||
|
// This Env might or might not be shared with other backups. To work
|
||||||
|
// around DBOptions::env being a raw pointer, this is a shared_ptr so
|
||||||
|
// that keeping either this BackupInfo, the BackupEngine, or a copy of
|
||||||
|
// this shared_ptr alive is sufficient to keep the Env alive for use by
|
||||||
|
// a read-only DB.
|
||||||
|
std::shared_ptr<Env> env_for_open; |
||||||
|
|
||||||
|
BackupInfo() {} |
||||||
|
|
||||||
|
BackupInfo(BackupID _backup_id, int64_t _timestamp, uint64_t _size, |
||||||
|
uint32_t _number_files, const std::string& _app_metadata) |
||||||
|
: backup_id(_backup_id), |
||||||
|
timestamp(_timestamp), |
||||||
|
size(_size), |
||||||
|
number_files(_number_files), |
||||||
|
app_metadata(_app_metadata) {} |
||||||
|
}; |
||||||
|
|
||||||
|
class BackupStatistics { |
||||||
|
public: |
||||||
|
BackupStatistics() { |
||||||
|
number_success_backup = 0; |
||||||
|
number_fail_backup = 0; |
||||||
|
} |
||||||
|
|
||||||
|
BackupStatistics(uint32_t _number_success_backup, |
||||||
|
uint32_t _number_fail_backup) |
||||||
|
: number_success_backup(_number_success_backup), |
||||||
|
number_fail_backup(_number_fail_backup) {} |
||||||
|
|
||||||
|
~BackupStatistics() {} |
||||||
|
|
||||||
|
void IncrementNumberSuccessBackup(); |
||||||
|
void IncrementNumberFailBackup(); |
||||||
|
|
||||||
|
uint32_t GetNumberSuccessBackup() const; |
||||||
|
uint32_t GetNumberFailBackup() const; |
||||||
|
|
||||||
|
std::string ToString() const; |
||||||
|
|
||||||
|
private: |
||||||
|
uint32_t number_success_backup; |
||||||
|
uint32_t number_fail_backup; |
||||||
|
}; |
||||||
|
|
||||||
|
// Read-only functions of a BackupEngine. (Restore writes to another directory
|
||||||
|
// not the backup directory.) See BackupEngine comments for details on
|
||||||
|
// safe concurrent operations.
|
||||||
|
class BackupEngineReadOnlyBase { |
||||||
|
public: |
||||||
|
virtual ~BackupEngineReadOnlyBase() {} |
||||||
|
|
||||||
|
// Returns info about the latest good backup in backup_info, or NotFound
|
||||||
|
// no good backup exists.
|
||||||
|
// Setting include_file_details=true provides information about each
|
||||||
|
// backed-up file in BackupInfo::file_details and more.
|
||||||
|
virtual Status GetLatestBackupInfo( |
||||||
|
BackupInfo* backup_info, bool include_file_details = false) const = 0; |
||||||
|
|
||||||
|
// Returns info about a specific backup in backup_info, or NotFound
|
||||||
|
// or Corruption status if the requested backup id does not exist or is
|
||||||
|
// known corrupt.
|
||||||
|
// Setting include_file_details=true provides information about each
|
||||||
|
// backed-up file in BackupInfo::file_details and more.
|
||||||
|
virtual Status GetBackupInfo(BackupID backup_id, BackupInfo* backup_info, |
||||||
|
bool include_file_details = false) const = 0; |
||||||
|
|
||||||
|
// Returns info about backups in backup_info
|
||||||
|
// Setting include_file_details=true provides information about each
|
||||||
|
// backed-up file in BackupInfo::file_details and more.
|
||||||
|
virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info, |
||||||
|
bool include_file_details = false) const = 0; |
||||||
|
|
||||||
|
// Returns info about corrupt backups in corrupt_backups.
|
||||||
|
// WARNING: Any write to the BackupEngine could trigger automatic
|
||||||
|
// GarbageCollect(), which could delete files that would be needed to
|
||||||
|
// manually recover a corrupt backup or to preserve an unrecognized (e.g.
|
||||||
|
// incompatible future version) backup.
|
||||||
|
virtual void GetCorruptedBackups( |
||||||
|
std::vector<BackupID>* corrupt_backup_ids) const = 0; |
||||||
|
|
||||||
|
// Restore to specified db_dir and wal_dir from backup_id.
|
||||||
|
virtual Status RestoreDBFromBackup(const RestoreOptions& options, |
||||||
|
BackupID backup_id, |
||||||
|
const std::string& db_dir, |
||||||
|
const std::string& wal_dir) const = 0; |
||||||
|
|
||||||
|
// keep for backward compatibility.
|
||||||
|
virtual Status RestoreDBFromBackup( |
||||||
|
BackupID backup_id, const std::string& db_dir, const std::string& wal_dir, |
||||||
|
const RestoreOptions& options = RestoreOptions()) const { |
||||||
|
return RestoreDBFromBackup(options, backup_id, db_dir, wal_dir); |
||||||
|
} |
||||||
|
|
||||||
|
// Like RestoreDBFromBackup but restores from latest non-corrupt backup_id
|
||||||
|
virtual Status RestoreDBFromLatestBackup( |
||||||
|
const RestoreOptions& options, const std::string& db_dir, |
||||||
|
const std::string& wal_dir) const = 0; |
||||||
|
|
||||||
|
// keep for backward compatibility.
|
||||||
|
virtual Status RestoreDBFromLatestBackup( |
||||||
|
const std::string& db_dir, const std::string& wal_dir, |
||||||
|
const RestoreOptions& options = RestoreOptions()) const { |
||||||
|
return RestoreDBFromLatestBackup(options, db_dir, wal_dir); |
||||||
|
} |
||||||
|
|
||||||
|
// If verify_with_checksum is true, this function
|
||||||
|
// inspects the current checksums and file sizes of backup files to see if
|
||||||
|
// they match our expectation.
|
||||||
|
//
|
||||||
|
// If verify_with_checksum is false, this function
|
||||||
|
// checks that each file exists and that the size of the file matches our
|
||||||
|
// expectation. It does not check file checksum.
|
||||||
|
//
|
||||||
|
// If this BackupEngine created the backup, it compares the files' current
|
||||||
|
// sizes (and current checksum) against the number of bytes written to
|
||||||
|
// them (and the checksum calculated) during creation.
|
||||||
|
// Otherwise, it compares the files' current sizes (and checksums) against
|
||||||
|
// their sizes (and checksums) when the BackupEngine was opened.
|
||||||
|
//
|
||||||
|
// Returns Status::OK() if all checks are good
|
||||||
|
virtual Status VerifyBackup(BackupID backup_id, |
||||||
|
bool verify_with_checksum = false) const = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
// Append-only functions of a BackupEngine. See BackupEngine comment for
|
||||||
|
// details on distinction between Append and Write operations and safe
|
||||||
|
// concurrent operations.
|
||||||
|
class BackupEngineAppendOnlyBase { |
||||||
|
public: |
||||||
|
virtual ~BackupEngineAppendOnlyBase() {} |
||||||
|
|
||||||
|
// same as CreateNewBackup, but stores extra application metadata.
|
||||||
|
virtual Status CreateNewBackupWithMetadata( |
||||||
|
const CreateBackupOptions& options, DB* db, |
||||||
|
const std::string& app_metadata, BackupID* new_backup_id = nullptr) = 0; |
||||||
|
|
||||||
|
// keep here for backward compatibility.
|
||||||
|
virtual Status CreateNewBackupWithMetadata( |
||||||
|
DB* db, const std::string& app_metadata, bool flush_before_backup = false, |
||||||
|
std::function<void()> progress_callback = []() {}) { |
||||||
|
CreateBackupOptions options; |
||||||
|
options.flush_before_backup = flush_before_backup; |
||||||
|
options.progress_callback = progress_callback; |
||||||
|
return CreateNewBackupWithMetadata(options, db, app_metadata); |
||||||
|
} |
||||||
|
|
||||||
|
// Captures the state of the database by creating a new (latest) backup.
|
||||||
|
// On success (OK status), the BackupID of the new backup is saved to
|
||||||
|
// *new_backup_id when not nullptr.
|
||||||
|
virtual Status CreateNewBackup(const CreateBackupOptions& options, DB* db, |
||||||
|
BackupID* new_backup_id = nullptr) { |
||||||
|
return CreateNewBackupWithMetadata(options, db, "", new_backup_id); |
||||||
|
} |
||||||
|
|
||||||
|
// keep here for backward compatibility.
|
||||||
|
virtual Status CreateNewBackup( |
||||||
|
DB* db, bool flush_before_backup = false, |
||||||
|
std::function<void()> progress_callback = []() {}) { |
||||||
|
CreateBackupOptions options; |
||||||
|
options.flush_before_backup = flush_before_backup; |
||||||
|
options.progress_callback = progress_callback; |
||||||
|
return CreateNewBackup(options, db); |
||||||
|
} |
||||||
|
|
||||||
|
// Call this from another thread if you want to stop the backup
|
||||||
|
// that is currently happening. It will return immediately, will
|
||||||
|
// not wait for the backup to stop.
|
||||||
|
// The backup will stop ASAP and the call to CreateNewBackup will
|
||||||
|
// return Status::Incomplete(). It will not clean up after itself, but
|
||||||
|
// the state will remain consistent. The state will be cleaned up the
|
||||||
|
// next time you call CreateNewBackup or GarbageCollect.
|
||||||
|
virtual void StopBackup() = 0; |
||||||
|
|
||||||
|
// Will delete any files left over from incomplete creation or deletion of
|
||||||
|
// a backup. This is not normally needed as those operations also clean up
|
||||||
|
// after prior incomplete calls to the same kind of operation (create or
|
||||||
|
// delete). This does not delete corrupt backups but can delete files that
|
||||||
|
// would be needed to manually recover a corrupt backup or to preserve an
|
||||||
|
// unrecognized (e.g. incompatible future version) backup.
|
||||||
|
// NOTE: This is not designed to delete arbitrary files added to the backup
|
||||||
|
// directory outside of BackupEngine, and clean-up is always subject to
|
||||||
|
// permissions on and availability of the underlying filesystem.
|
||||||
|
// NOTE2: For concurrency and interference purposes (see BackupEngine
|
||||||
|
// comment), GarbageCollect (GC) is like other Append operations, even
|
||||||
|
// though it seems different. Although GC can delete physical data, it does
|
||||||
|
// not delete any logical data read by Read operations. GC can interfere
|
||||||
|
// with Append or Write operations in another BackupEngine on the same
|
||||||
|
// backup_dir, because temporary files will be treated as obsolete and
|
||||||
|
// deleted.
|
||||||
|
virtual Status GarbageCollect() = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
// A backup engine for organizing and managing backups.
|
||||||
|
// This class is not user-extensible.
|
||||||
|
//
|
||||||
|
// This class declaration adds "Write" operations in addition to the
|
||||||
|
// operations from BackupEngineAppendOnlyBase and BackupEngineReadOnlyBase.
|
||||||
|
//
|
||||||
|
// # Concurrency between threads on the same BackupEngine* object
|
||||||
|
//
|
||||||
|
// As of version 6.20, BackupEngine* operations are generally thread-safe,
|
||||||
|
// using a read-write lock, though single-thread operation is still
|
||||||
|
// recommended to avoid TOCTOU bugs. Specifically, particular kinds of
|
||||||
|
// concurrent operations behave like this:
|
||||||
|
//
|
||||||
|
// op1\op2| Read | Append | Write
|
||||||
|
// -------|-------|--------|--------
|
||||||
|
// Read | conc | block | block
|
||||||
|
// Append | block | block | block
|
||||||
|
// Write | block | block | block
|
||||||
|
//
|
||||||
|
// conc = operations safely proceed concurrently
|
||||||
|
// block = one of the operations safely blocks until the other completes.
|
||||||
|
// There is generally no guarantee as to which completes first.
|
||||||
|
//
|
||||||
|
// StopBackup is the only operation that affects an ongoing operation.
|
||||||
|
//
|
||||||
|
// # Interleaving operations between BackupEngine* objects open on the
|
||||||
|
// same backup_dir
|
||||||
|
//
|
||||||
|
// It is recommended only to have one BackupEngine* object open for a given
|
||||||
|
// backup_dir, but it is possible to mix / interleave some operations
|
||||||
|
// (regardless of whether they are concurrent) with these caveats:
|
||||||
|
//
|
||||||
|
// op1\op2| Open | Read | Append | Write
|
||||||
|
// -------|--------|--------|--------|--------
|
||||||
|
// Open | conc | conc | atomic | unspec
|
||||||
|
// Read | conc | conc | old | unspec
|
||||||
|
// Append | atomic | old | unspec | unspec
|
||||||
|
// Write | unspec | unspec | unspec | unspec
|
||||||
|
//
|
||||||
|
// Special case: Open with destroy_old_data=true is really a Write
|
||||||
|
//
|
||||||
|
// conc = operations safely proceed, concurrently when applicable
|
||||||
|
// atomic = operations are effectively atomic; if a concurrent Append
|
||||||
|
// operation has not completed at some key point during Open, the
|
||||||
|
// opened BackupEngine* will never see the result of the Append op.
|
||||||
|
// old = Read operations do not include any state changes from other
|
||||||
|
// BackupEngine* objects; they return the state at their Open time.
|
||||||
|
// unspec = Behavior is unspecified, including possibly trashing the
|
||||||
|
// backup_dir, but is "memory safe" (no C++ undefined behavior)
|
||||||
|
//
|
||||||
|
class BackupEngine : public BackupEngineReadOnlyBase, |
||||||
|
public BackupEngineAppendOnlyBase { |
||||||
|
public: |
||||||
|
virtual ~BackupEngine() {} |
||||||
|
|
||||||
|
// BackupEngineOptions have to be the same as the ones used in previous
|
||||||
|
// BackupEngines for the same backup directory.
|
||||||
|
static Status Open(const BackupEngineOptions& options, Env* db_env, |
||||||
|
BackupEngine** backup_engine_ptr); |
||||||
|
|
||||||
|
// keep for backward compatibility.
|
||||||
|
static Status Open(Env* db_env, const BackupEngineOptions& options, |
||||||
|
BackupEngine** backup_engine_ptr) { |
||||||
|
return BackupEngine::Open(options, db_env, backup_engine_ptr); |
||||||
|
} |
||||||
|
|
||||||
|
// Deletes old backups, keeping latest num_backups_to_keep alive.
|
||||||
|
// See also DeleteBackup.
|
||||||
|
virtual Status PurgeOldBackups(uint32_t num_backups_to_keep) = 0; |
||||||
|
|
||||||
|
// Deletes a specific backup. If this operation (or PurgeOldBackups)
|
||||||
|
// is not completed due to crash, power failure, etc. the state
|
||||||
|
// will be cleaned up the next time you call DeleteBackup,
|
||||||
|
// PurgeOldBackups, or GarbageCollect.
|
||||||
|
virtual Status DeleteBackup(BackupID backup_id) = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
// A variant of BackupEngine that only allows "Read" operations. See
|
||||||
|
// BackupEngine comment for details. This class is not user-extensible.
|
||||||
|
class BackupEngineReadOnly : public BackupEngineReadOnlyBase { |
||||||
|
public: |
||||||
|
virtual ~BackupEngineReadOnly() {} |
||||||
|
|
||||||
|
static Status Open(const BackupEngineOptions& options, Env* db_env, |
||||||
|
BackupEngineReadOnly** backup_engine_ptr); |
||||||
|
// keep for backward compatibility.
|
||||||
|
static Status Open(Env* db_env, const BackupEngineOptions& options, |
||||||
|
BackupEngineReadOnly** backup_engine_ptr) { |
||||||
|
return BackupEngineReadOnly::Open(options, db_env, backup_engine_ptr); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -1,608 +1,26 @@ |
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
//
|
|
||||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
// This is a DEPRECATED header for API backward compatibility. Please
|
||||||
// Use of this source code is governed by a BSD-style license that can be
|
// use backup_engine.h.
|
||||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
||||||
|
|
||||||
#pragma once |
#pragma once |
||||||
#ifndef ROCKSDB_LITE |
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
// A legacy unnecessary include
|
||||||
#include <cinttypes> |
#include <cinttypes> |
||||||
#include <functional> |
|
||||||
#include <map> |
|
||||||
#include <string> |
|
||||||
#include <vector> |
|
||||||
|
|
||||||
#include "rocksdb/utilities/stackable_db.h" |
#include "rocksdb/utilities/backup_engine.h" |
||||||
|
|
||||||
#include "rocksdb/env.h" |
// A legacy unnecessary include
|
||||||
#include "rocksdb/options.h" |
#include "rocksdb/utilities/stackable_db.h" |
||||||
#include "rocksdb/status.h" |
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE { |
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
// The default DB file checksum function name.
|
using BackupableDBOptions = BackupEngineOptions; |
||||||
constexpr char kDbFileChecksumFuncName[] = "FileChecksumCrc32c"; |
|
||||||
// The default BackupEngine file checksum function name.
|
|
||||||
constexpr char kBackupFileChecksumFuncName[] = "crc32c"; |
|
||||||
|
|
||||||
struct BackupableDBOptions { |
|
||||||
// Where to keep the backup files. Has to be different than dbname_
|
|
||||||
// Best to set this to dbname_ + "/backups"
|
|
||||||
// Required
|
|
||||||
std::string backup_dir; |
|
||||||
|
|
||||||
// Backup Env object. It will be used for backup file I/O. If it's
|
|
||||||
// nullptr, backups will be written out using DBs Env. If it's
|
|
||||||
// non-nullptr, backup's I/O will be performed using this object.
|
|
||||||
// If you want to have backups on HDFS, use HDFS Env here!
|
|
||||||
// Default: nullptr
|
|
||||||
Env* backup_env; |
|
||||||
|
|
||||||
// share_table_files supports table and blob files.
|
|
||||||
//
|
|
||||||
// If share_table_files == true, the backup directory will share table and
|
|
||||||
// blob files among backups, to save space among backups of the same DB and to
|
|
||||||
// enable incremental backups by only copying new files.
|
|
||||||
// If share_table_files == false, each backup will be on its own and will not
|
|
||||||
// share any data with other backups.
|
|
||||||
//
|
|
||||||
// default: true
|
|
||||||
bool share_table_files; |
|
||||||
|
|
||||||
// Backup info and error messages will be written to info_log
|
|
||||||
// if non-nullptr.
|
|
||||||
// Default: nullptr
|
|
||||||
Logger* info_log; |
|
||||||
|
|
||||||
// If sync == true, we can guarantee you'll get consistent backup even
|
|
||||||
// on a machine crash/reboot. Backup process is slower with sync enabled.
|
|
||||||
// If sync == false, we don't guarantee anything on machine reboot. However,
|
|
||||||
// chances are some of the backups are consistent.
|
|
||||||
// Default: true
|
|
||||||
bool sync; |
|
||||||
|
|
||||||
// If true, it will delete whatever backups there are already
|
|
||||||
// Default: false
|
|
||||||
bool destroy_old_data; |
|
||||||
|
|
||||||
// If false, we won't backup log files. This option can be useful for backing
|
|
||||||
// up in-memory databases where log file are persisted, but table files are in
|
|
||||||
// memory.
|
|
||||||
// Default: true
|
|
||||||
bool backup_log_files; |
|
||||||
|
|
||||||
// Max bytes that can be transferred in a second during backup.
|
|
||||||
// If 0, go as fast as you can
|
|
||||||
// Default: 0
|
|
||||||
uint64_t backup_rate_limit; |
|
||||||
|
|
||||||
// Backup rate limiter. Used to control transfer speed for backup. If this is
|
|
||||||
// not null, backup_rate_limit is ignored.
|
|
||||||
// Default: nullptr
|
|
||||||
std::shared_ptr<RateLimiter> backup_rate_limiter{nullptr}; |
|
||||||
|
|
||||||
// Max bytes that can be transferred in a second during restore.
|
|
||||||
// If 0, go as fast as you can
|
|
||||||
// Default: 0
|
|
||||||
uint64_t restore_rate_limit; |
|
||||||
|
|
||||||
// Restore rate limiter. Used to control transfer speed during restore. If
|
|
||||||
// this is not null, restore_rate_limit is ignored.
|
|
||||||
// Default: nullptr
|
|
||||||
std::shared_ptr<RateLimiter> restore_rate_limiter{nullptr}; |
|
||||||
|
|
||||||
// share_files_with_checksum supports table and blob files.
|
|
||||||
//
|
|
||||||
// Only used if share_table_files is set to true. Setting to false is
|
|
||||||
// DEPRECATED and potentially dangerous because in that case BackupEngine
|
|
||||||
// can lose data if backing up databases with distinct or divergent
|
|
||||||
// history, for example if restoring from a backup other than the latest,
|
|
||||||
// writing to the DB, and creating another backup. Setting to true (default)
|
|
||||||
// prevents these issues by ensuring that different table files (SSTs) and
|
|
||||||
// blob files with the same number are treated as distinct. See
|
|
||||||
// share_files_with_checksum_naming and ShareFilesNaming.
|
|
||||||
//
|
|
||||||
// Default: true
|
|
||||||
bool share_files_with_checksum; |
|
||||||
|
|
||||||
// Up to this many background threads will copy files for CreateNewBackup()
|
|
||||||
// and RestoreDBFromBackup()
|
|
||||||
// Default: 1
|
|
||||||
int max_background_operations; |
|
||||||
|
|
||||||
// During backup user can get callback every time next
|
|
||||||
// callback_trigger_interval_size bytes being copied.
|
|
||||||
// Default: 4194304
|
|
||||||
uint64_t callback_trigger_interval_size; |
|
||||||
|
|
||||||
// For BackupEngineReadOnly, Open() will open at most this many of the
|
|
||||||
// latest non-corrupted backups.
|
|
||||||
//
|
|
||||||
// Note: this setting is ignored (behaves like INT_MAX) for any kind of
|
|
||||||
// writable BackupEngine because it would inhibit accounting for shared
|
|
||||||
// files for proper backup deletion, including purging any incompletely
|
|
||||||
// created backups on creation of a new backup.
|
|
||||||
//
|
|
||||||
// Default: INT_MAX
|
|
||||||
int max_valid_backups_to_open; |
|
||||||
|
|
||||||
// ShareFilesNaming describes possible naming schemes for backup
|
|
||||||
// table and blob file names when they are stored in the
|
|
||||||
// shared_checksum directory (i.e., both share_table_files and
|
|
||||||
// share_files_with_checksum are true).
|
|
||||||
enum ShareFilesNaming : uint32_t { |
|
||||||
// Backup blob filenames are <file_number>_<crc32c>_<file_size>.blob and
|
|
||||||
// backup SST filenames are <file_number>_<crc32c>_<file_size>.sst
|
|
||||||
// where <crc32c> is an unsigned decimal integer. This is the
|
|
||||||
// original/legacy naming scheme for share_files_with_checksum,
|
|
||||||
// with two problems:
|
|
||||||
// * At massive scale, collisions on this triple with different file
|
|
||||||
// contents is plausible.
|
|
||||||
// * Determining the name to use requires computing the checksum,
|
|
||||||
// so generally requires reading the whole file even if the file
|
|
||||||
// is already backed up.
|
|
||||||
//
|
|
||||||
// ** ONLY RECOMMENDED FOR PRESERVING OLD BEHAVIOR **
|
|
||||||
kLegacyCrc32cAndFileSize = 1U, |
|
||||||
|
|
||||||
// Backup SST filenames are <file_number>_s<db_session_id>.sst. This
|
|
||||||
// pair of values should be very strongly unique for a given SST file
|
|
||||||
// and easily determined before computing a checksum. The 's' indicates
|
|
||||||
// the value is a DB session id, not a checksum.
|
|
||||||
//
|
|
||||||
// Exceptions:
|
|
||||||
// * For blob files, kLegacyCrc32cAndFileSize is used as currently
|
|
||||||
// db_session_id is not supported by the blob file format.
|
|
||||||
// * For old SST files without a DB session id, kLegacyCrc32cAndFileSize
|
|
||||||
// will be used instead, matching the names assigned by RocksDB versions
|
|
||||||
// not supporting the newer naming scheme.
|
|
||||||
// * See also flags below.
|
|
||||||
kUseDbSessionId = 2U, |
|
||||||
|
|
||||||
kMaskNoNamingFlags = 0xffffU, |
|
||||||
|
|
||||||
// If not already part of the naming scheme, insert
|
|
||||||
// _<file_size>
|
|
||||||
// before .sst and .blob in the name. In case of user code actually parsing
|
|
||||||
// the last _<whatever> before the .sst and .blob as the file size, this
|
|
||||||
// preserves that feature of kLegacyCrc32cAndFileSize. In other words, this
|
|
||||||
// option makes official that unofficial feature of the backup metadata.
|
|
||||||
//
|
|
||||||
// We do not consider SST and blob file sizes to have sufficient entropy to
|
|
||||||
// contribute significantly to naming uniqueness.
|
|
||||||
kFlagIncludeFileSize = 1U << 31, |
|
||||||
|
|
||||||
kMaskNamingFlags = ~kMaskNoNamingFlags, |
|
||||||
}; |
|
||||||
|
|
||||||
// Naming option for share_files_with_checksum table and blob files. See
|
|
||||||
// ShareFilesNaming for details.
|
|
||||||
//
|
|
||||||
// Modifying this option cannot introduce a downgrade compatibility issue
|
|
||||||
// because RocksDB can read, restore, and delete backups using different file
|
|
||||||
// names, and it's OK for a backup directory to use a mixture of table and
|
|
||||||
// blob files naming schemes.
|
|
||||||
//
|
|
||||||
// However, modifying this option and saving more backups to the same
|
|
||||||
// directory can lead to the same file getting saved again to that
|
|
||||||
// directory, under the new shared name in addition to the old shared
|
|
||||||
// name.
|
|
||||||
//
|
|
||||||
// Default: kUseDbSessionId | kFlagIncludeFileSize
|
|
||||||
//
|
|
||||||
// Note: This option comes into effect only if both share_files_with_checksum
|
|
||||||
// and share_table_files are true.
|
|
||||||
ShareFilesNaming share_files_with_checksum_naming; |
|
||||||
|
|
||||||
void Dump(Logger* logger) const; |
|
||||||
|
|
||||||
explicit BackupableDBOptions( |
|
||||||
const std::string& _backup_dir, Env* _backup_env = nullptr, |
|
||||||
bool _share_table_files = true, Logger* _info_log = nullptr, |
|
||||||
bool _sync = true, bool _destroy_old_data = false, |
|
||||||
bool _backup_log_files = true, uint64_t _backup_rate_limit = 0, |
|
||||||
uint64_t _restore_rate_limit = 0, int _max_background_operations = 1, |
|
||||||
uint64_t _callback_trigger_interval_size = 4 * 1024 * 1024, |
|
||||||
int _max_valid_backups_to_open = INT_MAX, |
|
||||||
ShareFilesNaming _share_files_with_checksum_naming = |
|
||||||
static_cast<ShareFilesNaming>(kUseDbSessionId | kFlagIncludeFileSize)) |
|
||||||
: backup_dir(_backup_dir), |
|
||||||
backup_env(_backup_env), |
|
||||||
share_table_files(_share_table_files), |
|
||||||
info_log(_info_log), |
|
||||||
sync(_sync), |
|
||||||
destroy_old_data(_destroy_old_data), |
|
||||||
backup_log_files(_backup_log_files), |
|
||||||
backup_rate_limit(_backup_rate_limit), |
|
||||||
restore_rate_limit(_restore_rate_limit), |
|
||||||
share_files_with_checksum(true), |
|
||||||
max_background_operations(_max_background_operations), |
|
||||||
callback_trigger_interval_size(_callback_trigger_interval_size), |
|
||||||
max_valid_backups_to_open(_max_valid_backups_to_open), |
|
||||||
share_files_with_checksum_naming(_share_files_with_checksum_naming) { |
|
||||||
assert(share_table_files || !share_files_with_checksum); |
|
||||||
assert((share_files_with_checksum_naming & kMaskNoNamingFlags) != 0); |
|
||||||
} |
|
||||||
}; |
|
||||||
|
|
||||||
inline BackupableDBOptions::ShareFilesNaming operator&( |
|
||||||
BackupableDBOptions::ShareFilesNaming lhs, |
|
||||||
BackupableDBOptions::ShareFilesNaming rhs) { |
|
||||||
uint32_t l = static_cast<uint32_t>(lhs); |
|
||||||
uint32_t r = static_cast<uint32_t>(rhs); |
|
||||||
assert(r == BackupableDBOptions::kMaskNoNamingFlags || |
|
||||||
(r & BackupableDBOptions::kMaskNoNamingFlags) == 0); |
|
||||||
return static_cast<BackupableDBOptions::ShareFilesNaming>(l & r); |
|
||||||
} |
|
||||||
|
|
||||||
inline BackupableDBOptions::ShareFilesNaming operator|( |
|
||||||
BackupableDBOptions::ShareFilesNaming lhs, |
|
||||||
BackupableDBOptions::ShareFilesNaming rhs) { |
|
||||||
uint32_t l = static_cast<uint32_t>(lhs); |
|
||||||
uint32_t r = static_cast<uint32_t>(rhs); |
|
||||||
assert((r & BackupableDBOptions::kMaskNoNamingFlags) == 0); |
|
||||||
return static_cast<BackupableDBOptions::ShareFilesNaming>(l | r); |
|
||||||
} |
|
||||||
|
|
||||||
struct CreateBackupOptions { |
|
||||||
// Flush will always trigger if 2PC is enabled.
|
|
||||||
// If write-ahead logs are disabled, set flush_before_backup=true to
|
|
||||||
// avoid losing unflushed key/value pairs from the memtable.
|
|
||||||
bool flush_before_backup = false; |
|
||||||
|
|
||||||
// Callback for reporting progress, based on callback_trigger_interval_size.
|
|
||||||
std::function<void()> progress_callback = []() {}; |
|
||||||
|
|
||||||
// If false, background_thread_cpu_priority is ignored.
|
|
||||||
// Otherwise, the cpu priority can be decreased,
|
|
||||||
// if you try to increase the priority, the priority will not change.
|
|
||||||
// The initial priority of the threads is CpuPriority::kNormal,
|
|
||||||
// so you can decrease to priorities lower than kNormal.
|
|
||||||
bool decrease_background_thread_cpu_priority = false; |
|
||||||
CpuPriority background_thread_cpu_priority = CpuPriority::kNormal; |
|
||||||
}; |
|
||||||
|
|
||||||
struct RestoreOptions { |
|
||||||
// If true, restore won't overwrite the existing log files in wal_dir. It will
|
|
||||||
// also move all log files from archive directory to wal_dir. Use this option
|
|
||||||
// in combination with BackupableDBOptions::backup_log_files = false for
|
|
||||||
// persisting in-memory databases.
|
|
||||||
// Default: false
|
|
||||||
bool keep_log_files; |
|
||||||
|
|
||||||
explicit RestoreOptions(bool _keep_log_files = false) |
|
||||||
: keep_log_files(_keep_log_files) {} |
|
||||||
}; |
|
||||||
|
|
||||||
struct BackupFileInfo { |
|
||||||
// File name and path relative to the backup_dir directory.
|
|
||||||
std::string relative_filename; |
|
||||||
|
|
||||||
// Size of the file in bytes, not including filesystem overheads.
|
|
||||||
uint64_t size; |
|
||||||
}; |
|
||||||
|
|
||||||
typedef uint32_t BackupID; |
|
||||||
|
|
||||||
struct BackupInfo { |
|
||||||
BackupID backup_id = 0U; |
|
||||||
// Creation time, according to GetCurrentTime
|
|
||||||
int64_t timestamp = 0; |
|
||||||
|
|
||||||
// Total size in bytes (based on file payloads, not including filesystem
|
|
||||||
// overheads or backup meta file)
|
|
||||||
uint64_t size = 0U; |
|
||||||
|
|
||||||
// Number of backed up files, some of which might be shared with other
|
|
||||||
// backups. Does not include backup meta file.
|
|
||||||
uint32_t number_files = 0U; |
|
||||||
|
|
||||||
// Backup API user metadata
|
|
||||||
std::string app_metadata; |
|
||||||
|
|
||||||
// Backup file details, if requested with include_file_details=true
|
|
||||||
std::vector<BackupFileInfo> file_details; |
|
||||||
|
|
||||||
// DB "name" (a directory in the backup_env) for opening this backup as a
|
|
||||||
// read-only DB. This should also be used as the DBOptions::wal_dir, such
|
|
||||||
// as by default setting wal_dir="". See also env_for_open.
|
|
||||||
// This field is only set if include_file_details=true
|
|
||||||
std::string name_for_open; |
|
||||||
|
|
||||||
// An Env(+FileSystem) for opening this backup as a read-only DB, with
|
|
||||||
// DB::OpenForReadOnly or similar. This field is only set if
|
|
||||||
// include_file_details=true. (The FileSystem in this Env takes care
|
|
||||||
// of making shared backup files openable from the `name_for_open` DB
|
|
||||||
// directory.) See also name_for_open.
|
|
||||||
//
|
|
||||||
// This Env might or might not be shared with other backups. To work
|
|
||||||
// around DBOptions::env being a raw pointer, this is a shared_ptr so
|
|
||||||
// that keeping either this BackupInfo, the BackupEngine, or a copy of
|
|
||||||
// this shared_ptr alive is sufficient to keep the Env alive for use by
|
|
||||||
// a read-only DB.
|
|
||||||
std::shared_ptr<Env> env_for_open; |
|
||||||
|
|
||||||
BackupInfo() {} |
|
||||||
|
|
||||||
BackupInfo(BackupID _backup_id, int64_t _timestamp, uint64_t _size, |
|
||||||
uint32_t _number_files, const std::string& _app_metadata) |
|
||||||
: backup_id(_backup_id), |
|
||||||
timestamp(_timestamp), |
|
||||||
size(_size), |
|
||||||
number_files(_number_files), |
|
||||||
app_metadata(_app_metadata) {} |
|
||||||
}; |
|
||||||
|
|
||||||
class BackupStatistics { |
|
||||||
public: |
|
||||||
BackupStatistics() { |
|
||||||
number_success_backup = 0; |
|
||||||
number_fail_backup = 0; |
|
||||||
} |
|
||||||
|
|
||||||
BackupStatistics(uint32_t _number_success_backup, |
|
||||||
uint32_t _number_fail_backup) |
|
||||||
: number_success_backup(_number_success_backup), |
|
||||||
number_fail_backup(_number_fail_backup) {} |
|
||||||
|
|
||||||
~BackupStatistics() {} |
|
||||||
|
|
||||||
void IncrementNumberSuccessBackup(); |
|
||||||
void IncrementNumberFailBackup(); |
|
||||||
|
|
||||||
uint32_t GetNumberSuccessBackup() const; |
|
||||||
uint32_t GetNumberFailBackup() const; |
|
||||||
|
|
||||||
std::string ToString() const; |
|
||||||
|
|
||||||
private: |
|
||||||
uint32_t number_success_backup; |
|
||||||
uint32_t number_fail_backup; |
|
||||||
}; |
|
||||||
|
|
||||||
// Read-only functions of a BackupEngine. (Restore writes to another directory
|
|
||||||
// not the backup directory.) See BackupEngine comments for details on
|
|
||||||
// safe concurrent operations.
|
|
||||||
class BackupEngineReadOnlyBase { |
|
||||||
public: |
|
||||||
virtual ~BackupEngineReadOnlyBase() {} |
|
||||||
|
|
||||||
// Returns info about the latest good backup in backup_info, or NotFound
|
|
||||||
// no good backup exists.
|
|
||||||
// Setting include_file_details=true provides information about each
|
|
||||||
// backed-up file in BackupInfo::file_details and more.
|
|
||||||
virtual Status GetLatestBackupInfo( |
|
||||||
BackupInfo* backup_info, bool include_file_details = false) const = 0; |
|
||||||
|
|
||||||
// Returns info about a specific backup in backup_info, or NotFound
|
|
||||||
// or Corruption status if the requested backup id does not exist or is
|
|
||||||
// known corrupt.
|
|
||||||
// Setting include_file_details=true provides information about each
|
|
||||||
// backed-up file in BackupInfo::file_details and more.
|
|
||||||
virtual Status GetBackupInfo(BackupID backup_id, BackupInfo* backup_info, |
|
||||||
bool include_file_details = false) const = 0; |
|
||||||
|
|
||||||
// Returns info about backups in backup_info
|
|
||||||
// Setting include_file_details=true provides information about each
|
|
||||||
// backed-up file in BackupInfo::file_details and more.
|
|
||||||
virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info, |
|
||||||
bool include_file_details = false) const = 0; |
|
||||||
|
|
||||||
// Returns info about corrupt backups in corrupt_backups.
|
|
||||||
// WARNING: Any write to the BackupEngine could trigger automatic
|
|
||||||
// GarbageCollect(), which could delete files that would be needed to
|
|
||||||
// manually recover a corrupt backup or to preserve an unrecognized (e.g.
|
|
||||||
// incompatible future version) backup.
|
|
||||||
virtual void GetCorruptedBackups( |
|
||||||
std::vector<BackupID>* corrupt_backup_ids) const = 0; |
|
||||||
|
|
||||||
// Restore to specified db_dir and wal_dir from backup_id.
|
|
||||||
virtual Status RestoreDBFromBackup(const RestoreOptions& options, |
|
||||||
BackupID backup_id, |
|
||||||
const std::string& db_dir, |
|
||||||
const std::string& wal_dir) const = 0; |
|
||||||
|
|
||||||
// keep for backward compatibility.
|
|
||||||
virtual Status RestoreDBFromBackup( |
|
||||||
BackupID backup_id, const std::string& db_dir, const std::string& wal_dir, |
|
||||||
const RestoreOptions& options = RestoreOptions()) const { |
|
||||||
return RestoreDBFromBackup(options, backup_id, db_dir, wal_dir); |
|
||||||
} |
|
||||||
|
|
||||||
// Like RestoreDBFromBackup but restores from latest non-corrupt backup_id
|
|
||||||
virtual Status RestoreDBFromLatestBackup( |
|
||||||
const RestoreOptions& options, const std::string& db_dir, |
|
||||||
const std::string& wal_dir) const = 0; |
|
||||||
|
|
||||||
// keep for backward compatibility.
|
|
||||||
virtual Status RestoreDBFromLatestBackup( |
|
||||||
const std::string& db_dir, const std::string& wal_dir, |
|
||||||
const RestoreOptions& options = RestoreOptions()) const { |
|
||||||
return RestoreDBFromLatestBackup(options, db_dir, wal_dir); |
|
||||||
} |
|
||||||
|
|
||||||
// If verify_with_checksum is true, this function
|
|
||||||
// inspects the current checksums and file sizes of backup files to see if
|
|
||||||
// they match our expectation.
|
|
||||||
//
|
|
||||||
// If verify_with_checksum is false, this function
|
|
||||||
// checks that each file exists and that the size of the file matches our
|
|
||||||
// expectation. It does not check file checksum.
|
|
||||||
//
|
|
||||||
// If this BackupEngine created the backup, it compares the files' current
|
|
||||||
// sizes (and current checksum) against the number of bytes written to
|
|
||||||
// them (and the checksum calculated) during creation.
|
|
||||||
// Otherwise, it compares the files' current sizes (and checksums) against
|
|
||||||
// their sizes (and checksums) when the BackupEngine was opened.
|
|
||||||
//
|
|
||||||
// Returns Status::OK() if all checks are good
|
|
||||||
virtual Status VerifyBackup(BackupID backup_id, |
|
||||||
bool verify_with_checksum = false) const = 0; |
|
||||||
}; |
|
||||||
|
|
||||||
// Append-only functions of a BackupEngine. See BackupEngine comment for
|
|
||||||
// details on distinction between Append and Write operations and safe
|
|
||||||
// concurrent operations.
|
|
||||||
class BackupEngineAppendOnlyBase { |
|
||||||
public: |
|
||||||
virtual ~BackupEngineAppendOnlyBase() {} |
|
||||||
|
|
||||||
// same as CreateNewBackup, but stores extra application metadata.
|
|
||||||
virtual Status CreateNewBackupWithMetadata( |
|
||||||
const CreateBackupOptions& options, DB* db, |
|
||||||
const std::string& app_metadata, BackupID* new_backup_id = nullptr) = 0; |
|
||||||
|
|
||||||
// keep here for backward compatibility.
|
|
||||||
virtual Status CreateNewBackupWithMetadata( |
|
||||||
DB* db, const std::string& app_metadata, bool flush_before_backup = false, |
|
||||||
std::function<void()> progress_callback = []() {}) { |
|
||||||
CreateBackupOptions options; |
|
||||||
options.flush_before_backup = flush_before_backup; |
|
||||||
options.progress_callback = progress_callback; |
|
||||||
return CreateNewBackupWithMetadata(options, db, app_metadata); |
|
||||||
} |
|
||||||
|
|
||||||
// Captures the state of the database by creating a new (latest) backup.
|
|
||||||
// On success (OK status), the BackupID of the new backup is saved to
|
|
||||||
// *new_backup_id when not nullptr.
|
|
||||||
virtual Status CreateNewBackup(const CreateBackupOptions& options, DB* db, |
|
||||||
BackupID* new_backup_id = nullptr) { |
|
||||||
return CreateNewBackupWithMetadata(options, db, "", new_backup_id); |
|
||||||
} |
|
||||||
|
|
||||||
// keep here for backward compatibility.
|
|
||||||
virtual Status CreateNewBackup(DB* db, bool flush_before_backup = false, |
|
||||||
std::function<void()> progress_callback = |
|
||||||
[]() {}) { |
|
||||||
CreateBackupOptions options; |
|
||||||
options.flush_before_backup = flush_before_backup; |
|
||||||
options.progress_callback = progress_callback; |
|
||||||
return CreateNewBackup(options, db); |
|
||||||
} |
|
||||||
|
|
||||||
// Call this from another thread if you want to stop the backup
|
|
||||||
// that is currently happening. It will return immediately, will
|
|
||||||
// not wait for the backup to stop.
|
|
||||||
// The backup will stop ASAP and the call to CreateNewBackup will
|
|
||||||
// return Status::Incomplete(). It will not clean up after itself, but
|
|
||||||
// the state will remain consistent. The state will be cleaned up the
|
|
||||||
// next time you call CreateNewBackup or GarbageCollect.
|
|
||||||
virtual void StopBackup() = 0; |
|
||||||
|
|
||||||
// Will delete any files left over from incomplete creation or deletion of
|
|
||||||
// a backup. This is not normally needed as those operations also clean up
|
|
||||||
// after prior incomplete calls to the same kind of operation (create or
|
|
||||||
// delete). This does not delete corrupt backups but can delete files that
|
|
||||||
// would be needed to manually recover a corrupt backup or to preserve an
|
|
||||||
// unrecognized (e.g. incompatible future version) backup.
|
|
||||||
// NOTE: This is not designed to delete arbitrary files added to the backup
|
|
||||||
// directory outside of BackupEngine, and clean-up is always subject to
|
|
||||||
// permissions on and availability of the underlying filesystem.
|
|
||||||
// NOTE2: For concurrency and interference purposes (see BackupEngine
|
|
||||||
// comment), GarbageCollect (GC) is like other Append operations, even
|
|
||||||
// though it seems different. Although GC can delete physical data, it does
|
|
||||||
// not delete any logical data read by Read operations. GC can interfere
|
|
||||||
// with Append or Write operations in another BackupEngine on the same
|
|
||||||
// backup_dir, because temporary files will be treated as obsolete and
|
|
||||||
// deleted.
|
|
||||||
virtual Status GarbageCollect() = 0; |
|
||||||
}; |
|
||||||
|
|
||||||
// A backup engine for organizing and managing backups.
|
|
||||||
// This class is not user-extensible.
|
|
||||||
//
|
|
||||||
// This class declaration adds "Write" operations in addition to the
|
|
||||||
// operations from BackupEngineAppendOnlyBase and BackupEngineReadOnlyBase.
|
|
||||||
//
|
|
||||||
// # Concurrency between threads on the same BackupEngine* object
|
|
||||||
//
|
|
||||||
// As of version 6.20, BackupEngine* operations are generally thread-safe,
|
|
||||||
// using a read-write lock, though single-thread operation is still
|
|
||||||
// recommended to avoid TOCTOU bugs. Specifically, particular kinds of
|
|
||||||
// concurrent operations behave like this:
|
|
||||||
//
|
|
||||||
// op1\op2| Read | Append | Write
|
|
||||||
// -------|-------|--------|--------
|
|
||||||
// Read | conc | block | block
|
|
||||||
// Append | block | block | block
|
|
||||||
// Write | block | block | block
|
|
||||||
//
|
|
||||||
// conc = operations safely proceed concurrently
|
|
||||||
// block = one of the operations safely blocks until the other completes.
|
|
||||||
// There is generally no guarantee as to which completes first.
|
|
||||||
//
|
|
||||||
// StopBackup is the only operation that affects an ongoing operation.
|
|
||||||
//
|
|
||||||
// # Interleaving operations between BackupEngine* objects open on the
|
|
||||||
// same backup_dir
|
|
||||||
//
|
|
||||||
// It is recommended only to have one BackupEngine* object open for a given
|
|
||||||
// backup_dir, but it is possible to mix / interleave some operations
|
|
||||||
// (regardless of whether they are concurrent) with these caveats:
|
|
||||||
//
|
|
||||||
// op1\op2| Open | Read | Append | Write
|
|
||||||
// -------|--------|--------|--------|--------
|
|
||||||
// Open | conc | conc | atomic | unspec
|
|
||||||
// Read | conc | conc | old | unspec
|
|
||||||
// Append | atomic | old | unspec | unspec
|
|
||||||
// Write | unspec | unspec | unspec | unspec
|
|
||||||
//
|
|
||||||
// Special case: Open with destroy_old_data=true is really a Write
|
|
||||||
//
|
|
||||||
// conc = operations safely proceed, concurrently when applicable
|
|
||||||
// atomic = operations are effectively atomic; if a concurrent Append
|
|
||||||
// operation has not completed at some key point during Open, the
|
|
||||||
// opened BackupEngine* will never see the result of the Append op.
|
|
||||||
// old = Read operations do not include any state changes from other
|
|
||||||
// BackupEngine* objects; they return the state at their Open time.
|
|
||||||
// unspec = Behavior is unspecified, including possibly trashing the
|
|
||||||
// backup_dir, but is "memory safe" (no C++ undefined behavior)
|
|
||||||
//
|
|
||||||
class BackupEngine : public BackupEngineReadOnlyBase, |
|
||||||
public BackupEngineAppendOnlyBase { |
|
||||||
public: |
|
||||||
virtual ~BackupEngine() {} |
|
||||||
|
|
||||||
// BackupableDBOptions have to be the same as the ones used in previous
|
|
||||||
// BackupEngines for the same backup directory.
|
|
||||||
static Status Open(const BackupableDBOptions& options, Env* db_env, |
|
||||||
BackupEngine** backup_engine_ptr); |
|
||||||
|
|
||||||
// keep for backward compatibility.
|
|
||||||
static Status Open(Env* db_env, const BackupableDBOptions& options, |
|
||||||
BackupEngine** backup_engine_ptr) { |
|
||||||
return BackupEngine::Open(options, db_env, backup_engine_ptr); |
|
||||||
} |
|
||||||
|
|
||||||
// Deletes old backups, keeping latest num_backups_to_keep alive.
|
|
||||||
// See also DeleteBackup.
|
|
||||||
virtual Status PurgeOldBackups(uint32_t num_backups_to_keep) = 0; |
|
||||||
|
|
||||||
// Deletes a specific backup. If this operation (or PurgeOldBackups)
|
|
||||||
// is not completed due to crash, power failure, etc. the state
|
|
||||||
// will be cleaned up the next time you call DeleteBackup,
|
|
||||||
// PurgeOldBackups, or GarbageCollect.
|
|
||||||
virtual Status DeleteBackup(BackupID backup_id) = 0; |
|
||||||
}; |
|
||||||
|
|
||||||
// A variant of BackupEngine that only allows "Read" operations. See
|
|
||||||
// BackupEngine comment for details. This class is not user-extensible.
|
|
||||||
class BackupEngineReadOnly : public BackupEngineReadOnlyBase { |
|
||||||
public: |
|
||||||
virtual ~BackupEngineReadOnly() {} |
|
||||||
|
|
||||||
static Status Open(const BackupableDBOptions& options, Env* db_env, |
|
||||||
BackupEngineReadOnly** backup_engine_ptr); |
|
||||||
// keep for backward compatibility.
|
|
||||||
static Status Open(Env* db_env, const BackupableDBOptions& options, |
|
||||||
BackupEngineReadOnly** backup_engine_ptr) { |
|
||||||
return BackupEngineReadOnly::Open(options, db_env, backup_engine_ptr); |
|
||||||
} |
|
||||||
}; |
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|||||||
Loading…
Reference in new issue