NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace Status with IOStatus in the backupable_db (#8820)

Browse Source 
Summary:
In order to populate the IOStatus up to the higher level, replace some of the Status to IOStatus.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8820

Test Plan: make check

Reviewed By: pdillinger

Differential Revision: D30967215

Pulled By: zhichao-cao

fbshipit-source-id: ccf9d5cfbd9d3de047c464aaa85f9fa43b474903
main
Zhichao Cao 3 years ago committed by Facebook GitHub Bot
parent 5c92aa38ea
commit 82e7631de6
6 changed files with 473 additions and 438 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 28
      file/line_file_reader.cc
  2. 12
      file/line_file_reader.h
  3. 26
      file/sequence_file_reader.cc
  4. 12
      file/sequence_file_reader.h
  5. 29
      include/rocksdb/utilities/backup_engine.h
  6. 804
      utilities/backupable/backupable_db.cc

28
file/line_file_reader.cc
Unescape View File

@ -11,24 +11,24 @@
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
Status LineFileReader::Create(const std::shared_ptr<FileSystem>& fs, IOStatus LineFileReader::Create(const std::shared_ptr<FileSystem>& fs,
const std::string& fname, const std::string& fname,
const FileOptions& file_opts, const FileOptions& file_opts,
std::unique_ptr<LineFileReader>* reader, std::unique_ptr<LineFileReader>* reader,
IODebugContext* dbg) { IODebugContext* dbg) {
std::unique_ptr<FSSequentialFile> file; std::unique_ptr<FSSequentialFile> file;
Status s = fs->NewSequentialFile(fname, file_opts, &file, dbg); IOStatus io_s = fs->NewSequentialFile(fname, file_opts, &file, dbg);
if (s.ok()) { if (io_s.ok()) {
reader->reset(new LineFileReader(std::move(file), fname)); reader->reset(new LineFileReader(std::move(file), fname));
} }
return s; return io_s;
} }
bool LineFileReader::ReadLine(std::string* out) { bool LineFileReader::ReadLine(std::string* out) {
assert(out); assert(out);
if (!status_.ok()) { if (!io_status_.ok()) {
// Status should be checked (or permit unchecked) any time we return false. // Status should be checked (or permit unchecked) any time we return false.
status_.MustCheck(); io_status_.MustCheck();
return false; return false;
} }
out->clear(); out->clear();
@ -44,16 +44,16 @@ bool LineFileReader::ReadLine(std::string* out) {
return true; return true;
} }
if (at_eof_) { if (at_eof_) {
status_.MustCheck(); io_status_.MustCheck();
return false; return false;
} }
// else flush and reload buffer // else flush and reload buffer
out->append(buf_begin_, buf_end_ - buf_begin_); out->append(buf_begin_, buf_end_ - buf_begin_);
Slice result; Slice result;
status_ = sfr_.Read(buf_.size(), &result, buf_.data()); io_status_ = sfr_.Read(buf_.size(), &result, buf_.data());
IOSTATS_ADD(bytes_read, result.size()); IOSTATS_ADD(bytes_read, result.size());
if (!status_.ok()) { if (!io_status_.ok()) {
status_.MustCheck(); io_status_.MustCheck();
return false; return false;
} }
if (result.size() != buf_.size()) { if (result.size() != buf_.size()) {

12
file/line_file_reader.h
Unescape View File

@ -17,7 +17,7 @@ class LineFileReader {
private: private:
std::array<char, 8192> buf_; std::array<char, 8192> buf_;
SequentialFileReader sfr_; SequentialFileReader sfr_;
Status status_; IOStatus io_status_;
const char* buf_begin_ = buf_.data(); const char* buf_begin_ = buf_.data();
const char* buf_end_ = buf_.data(); const char* buf_end_ = buf_.data();
size_t line_number_ = 0; size_t line_number_ = 0;
@ -29,10 +29,10 @@ class LineFileReader {
explicit LineFileReader(Args&&... args) explicit LineFileReader(Args&&... args)
: sfr_(std::forward<Args&&>(args)...) {} : sfr_(std::forward<Args&&>(args)...) {}
static Status Create(const std::shared_ptr<FileSystem>& fs, static IOStatus Create(const std::shared_ptr<FileSystem>& fs,
const std::string& fname, const FileOptions& file_opts, const std::string& fname, const FileOptions& file_opts,
std::unique_ptr<LineFileReader>* reader, std::unique_ptr<LineFileReader>* reader,
IODebugContext* dbg); IODebugContext* dbg);
LineFileReader(const LineFileReader&) = delete; LineFileReader(const LineFileReader&) = delete;
LineFileReader& operator=(const LineFileReader&) = delete; LineFileReader& operator=(const LineFileReader&) = delete;
@ -53,7 +53,7 @@ class LineFileReader {
// Returns any error encountered during read. The error is considered // Returns any error encountered during read. The error is considered
// permanent and no retry or recovery is attempted with the same // permanent and no retry or recovery is attempted with the same
// LineFileReader. // LineFileReader.
const Status& GetStatus() const { return status_; } const IOStatus& GetStatus() const { return io_status_; }
}; };
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

26
file/sequence_file_reader.cc
Unescape View File

@ -22,20 +22,20 @@
#include "util/rate_limiter.h" #include "util/rate_limiter.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
Status SequentialFileReader::Create( IOStatus SequentialFileReader::Create(
const std::shared_ptr<FileSystem>& fs, const std::string& fname, const std::shared_ptr<FileSystem>& fs, const std::string& fname,
const FileOptions& file_opts, std::unique_ptr<SequentialFileReader>* reader, const FileOptions& file_opts, std::unique_ptr<SequentialFileReader>* reader,
IODebugContext* dbg) { IODebugContext* dbg) {
std::unique_ptr<FSSequentialFile> file; std::unique_ptr<FSSequentialFile> file;
Status s = fs->NewSequentialFile(fname, file_opts, &file, dbg); IOStatus io_s = fs->NewSequentialFile(fname, file_opts, &file, dbg);
if (s.ok()) { if (io_s.ok()) {
reader->reset(new SequentialFileReader(std::move(file), fname)); reader->reset(new SequentialFileReader(std::move(file), fname));
} }
return s; return io_s;
} }
Status SequentialFileReader::Read(size_t n, Slice* result, char* scratch) { IOStatus SequentialFileReader::Read(size_t n, Slice* result, char* scratch) {
Status s; IOStatus io_s;
if (use_direct_io()) { if (use_direct_io()) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
size_t offset = offset_.fetch_add(n); size_t offset = offset_.fetch_add(n);
@ -48,9 +48,9 @@ Status SequentialFileReader::Read(size_t n, Slice* result, char* scratch) {
buf.Alignment(alignment); buf.Alignment(alignment);
buf.AllocateNewBuffer(size); buf.AllocateNewBuffer(size);
Slice tmp; Slice tmp;
s = file_->PositionedRead(aligned_offset, size, IOOptions(), &tmp, io_s = file_->PositionedRead(aligned_offset, size, IOOptions(), &tmp,
buf.BufferStart(), nullptr); buf.BufferStart(), nullptr);
if (s.ok() && offset_advance < tmp.size()) { if (io_s.ok() && offset_advance < tmp.size()) {
buf.Size(tmp.size()); buf.Size(tmp.size());
r = buf.Read(scratch, offset_advance, r = buf.Read(scratch, offset_advance,
std::min(tmp.size() - offset_advance, n)); std::min(tmp.size() - offset_advance, n));
@ -58,17 +58,17 @@ Status SequentialFileReader::Read(size_t n, Slice* result, char* scratch) {
*result = Slice(scratch, r); *result = Slice(scratch, r);
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
} else { } else {
s = file_->Read(n, IOOptions(), result, scratch, nullptr); io_s = file_->Read(n, IOOptions(), result, scratch, nullptr);
} }
IOSTATS_ADD(bytes_read, result->size()); IOSTATS_ADD(bytes_read, result->size());
return s; return io_s;
} }
Status SequentialFileReader::Skip(uint64_t n) { IOStatus SequentialFileReader::Skip(uint64_t n) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
if (use_direct_io()) { if (use_direct_io()) {
offset_ += static_cast<size_t>(n); offset_ += static_cast<size_t>(n);
return Status::OK(); return IOStatus::OK();
} }
#endif // !ROCKSDB_LITE #endif // !ROCKSDB_LITE
return file_->Skip(n); return file_->Skip(n);

12
file/sequence_file_reader.h
Unescape View File

@ -41,17 +41,17 @@ class SequentialFileReader {
: file_name_(_file_name), : file_name_(_file_name),
file_(NewReadaheadSequentialFile(std::move(_file), _readahead_size), file_(NewReadaheadSequentialFile(std::move(_file), _readahead_size),
io_tracer, _file_name) {} io_tracer, _file_name) {}
static Status Create(const std::shared_ptr<FileSystem>& fs, static IOStatus Create(const std::shared_ptr<FileSystem>& fs,
const std::string& fname, const FileOptions& file_opts, const std::string& fname, const FileOptions& file_opts,
std::unique_ptr<SequentialFileReader>* reader, std::unique_ptr<SequentialFileReader>* reader,
IODebugContext* dbg); IODebugContext* dbg);
SequentialFileReader(const SequentialFileReader&) = delete; SequentialFileReader(const SequentialFileReader&) = delete;
SequentialFileReader& operator=(const SequentialFileReader&) = delete; SequentialFileReader& operator=(const SequentialFileReader&) = delete;
Status Read(size_t n, Slice* result, char* scratch); IOStatus Read(size_t n, Slice* result, char* scratch);
Status Skip(uint64_t n); IOStatus Skip(uint64_t n);
FSSequentialFile* file() { return file_.get(); } FSSequentialFile* file() { return file_.get(); }

29
include/rocksdb/utilities/backup_engine.h
Unescape View File

@ -17,6 +17,7 @@
#include <vector> #include <vector>
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/io_status.h"
#include "rocksdb/options.h" #include "rocksdb/options.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
@ -406,25 +407,25 @@ class BackupEngineReadOnlyBase {
std::vector<BackupID>* corrupt_backup_ids) const = 0; std::vector<BackupID>* corrupt_backup_ids) const = 0;
// Restore to specified db_dir and wal_dir from backup_id. // Restore to specified db_dir and wal_dir from backup_id.
virtual Status RestoreDBFromBackup(const RestoreOptions& options, virtual IOStatus RestoreDBFromBackup(const RestoreOptions& options,
BackupID backup_id, BackupID backup_id,
const std::string& db_dir, const std::string& db_dir,
const std::string& wal_dir) const = 0; const std::string& wal_dir) const = 0;
// keep for backward compatibility. // keep for backward compatibility.
virtual Status RestoreDBFromBackup( virtual IOStatus RestoreDBFromBackup(
BackupID backup_id, const std::string& db_dir, const std::string& wal_dir, BackupID backup_id, const std::string& db_dir, const std::string& wal_dir,
const RestoreOptions& options = RestoreOptions()) const { const RestoreOptions& options = RestoreOptions()) const {
return RestoreDBFromBackup(options, backup_id, db_dir, wal_dir); return RestoreDBFromBackup(options, backup_id, db_dir, wal_dir);
} }
// Like RestoreDBFromBackup but restores from latest non-corrupt backup_id // Like RestoreDBFromBackup but restores from latest non-corrupt backup_id
virtual Status RestoreDBFromLatestBackup( virtual IOStatus RestoreDBFromLatestBackup(
const RestoreOptions& options, const std::string& db_dir, const RestoreOptions& options, const std::string& db_dir,
const std::string& wal_dir) const = 0; const std::string& wal_dir) const = 0;
// keep for backward compatibility. // keep for backward compatibility.
virtual Status RestoreDBFromLatestBackup( virtual IOStatus RestoreDBFromLatestBackup(
const std::string& db_dir, const std::string& wal_dir, const std::string& db_dir, const std::string& wal_dir,
const RestoreOptions& options = RestoreOptions()) const { const RestoreOptions& options = RestoreOptions()) const {
return RestoreDBFromLatestBackup(options, db_dir, wal_dir); return RestoreDBFromLatestBackup(options, db_dir, wal_dir);
@ -445,8 +446,8 @@ class BackupEngineReadOnlyBase {
// their sizes (and checksums) when the BackupEngine was opened. // their sizes (and checksums) when the BackupEngine was opened.
// //
// Returns Status::OK() if all checks are good // Returns Status::OK() if all checks are good
virtual Status VerifyBackup(BackupID backup_id, virtual IOStatus VerifyBackup(BackupID backup_id,
bool verify_with_checksum = false) const = 0; bool verify_with_checksum = false) const = 0;
}; };
// Append-only functions of a BackupEngine. See BackupEngine comment for // Append-only functions of a BackupEngine. See BackupEngine comment for
@ -457,12 +458,12 @@ class BackupEngineAppendOnlyBase {
virtual ~BackupEngineAppendOnlyBase() {} virtual ~BackupEngineAppendOnlyBase() {}
// same as CreateNewBackup, but stores extra application metadata. // same as CreateNewBackup, but stores extra application metadata.
virtual Status CreateNewBackupWithMetadata( virtual IOStatus CreateNewBackupWithMetadata(
const CreateBackupOptions& options, DB* db, const CreateBackupOptions& options, DB* db,
const std::string& app_metadata, BackupID* new_backup_id = nullptr) = 0; const std::string& app_metadata, BackupID* new_backup_id = nullptr) = 0;
// keep here for backward compatibility. // keep here for backward compatibility.
virtual Status CreateNewBackupWithMetadata( virtual IOStatus CreateNewBackupWithMetadata(
DB* db, const std::string& app_metadata, bool flush_before_backup = false, DB* db, const std::string& app_metadata, bool flush_before_backup = false,
std::function<void()> progress_callback = []() {}) { std::function<void()> progress_callback = []() {}) {
CreateBackupOptions options; CreateBackupOptions options;
@ -514,7 +515,7 @@ class BackupEngineAppendOnlyBase {
// with Append or Write operations in another BackupEngine on the same // with Append or Write operations in another BackupEngine on the same
// backup_dir, because temporary files will be treated as obsolete and // backup_dir, because temporary files will be treated as obsolete and
// deleted. // deleted.
virtual Status GarbageCollect() = 0; virtual IOStatus GarbageCollect() = 0;
}; };
// A backup engine for organizing and managing backups. // A backup engine for organizing and managing backups.
@ -585,13 +586,13 @@ class BackupEngine : public BackupEngineReadOnlyBase,
// Deletes old backups, keeping latest num_backups_to_keep alive. // Deletes old backups, keeping latest num_backups_to_keep alive.
// See also DeleteBackup. // See also DeleteBackup.
virtual Status PurgeOldBackups(uint32_t num_backups_to_keep) = 0; virtual IOStatus PurgeOldBackups(uint32_t num_backups_to_keep) = 0;
// Deletes a specific backup. If this operation (or PurgeOldBackups) // Deletes a specific backup. If this operation (or PurgeOldBackups)
// is not completed due to crash, power failure, etc. the state // is not completed due to crash, power failure, etc. the state
// will be cleaned up the next time you call DeleteBackup, // will be cleaned up the next time you call DeleteBackup,
// PurgeOldBackups, or GarbageCollect. // PurgeOldBackups, or GarbageCollect.
virtual Status DeleteBackup(BackupID backup_id) = 0; virtual IOStatus DeleteBackup(BackupID backup_id) = 0;
}; };
// A variant of BackupEngine that only allows "Read" operations. See // A variant of BackupEngine that only allows "Read" operations. See

804
utilities/backupable/backupable_db.cc
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save