NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improved FileExists API

Browse Source 
Summary: Add new CheckFileExists method.  Considered changing the FileExists api but didn't want to break anyone's builds.

Test Plan: unit tests

Reviewers: yhchiang, igor, sdong

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D42003
main
agiardullo 9 years ago
parent 9f1de95187
commit 064294081b
23 changed files with 191 additions and 96 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      HISTORY.md
  2. 2
      db/compact_files_test.cc
  3. 15
      db/db_impl.cc
  4. 6
      db/db_test.cc
  5. 12
      db/deletefile_test.cc
  6. 8
      db/fault_injection_test.cc
  7. 14
      db/wal_manager.cc
  8. 6
      hdfs/env_hdfs.h
  9. 10
      include/rocksdb/env.h
  10. 10
      include/rocksdb/status.h
  11. 5
      port/win/env_win.cc
  12. 3
      table/cuckoo_table_reader_test.cc
  13. 6
      util/auto_roll_logger_test.cc
  14. 12
      util/env_hdfs.cc
  15. 28
      util/env_posix.cc
  16. 8
      util/memenv.cc
  17. 10
      util/memenv_test.cc
  18. 8
      util/mock_env.cc
  19. 2
      util/mock_env.h
  20. 10
      util/mock_env_test.cc
  21. 31
      utilities/backupable/backupable_db.cc
  22. 74
      utilities/backupable/backupable_db_test.cc
  23. 6
      utilities/checkpoint/checkpoint.cc

1
HISTORY.md
Unescape View File

@ -7,6 +7,7 @@
* Deprecated purge_redundant_kvs_while_flush option. * Deprecated purge_redundant_kvs_while_flush option.
* Removed BackupEngine::NewBackupEngine() and NewReadOnlyBackupEngine() that were deprecated in RocksDB 3.8. Please use BackupEngine::Open() instead. * Removed BackupEngine::NewBackupEngine() and NewReadOnlyBackupEngine() that were deprecated in RocksDB 3.8. Please use BackupEngine::Open() instead.
* Deprecated Compaction Filter V2. We are not aware of any existing use-cases. If you use this filter, your compile will break with RocksDB 3.13. Please let us know if you use it and we'll put it back in RocksDB 3.14. * Deprecated Compaction Filter V2. We are not aware of any existing use-cases. If you use this filter, your compile will break with RocksDB 3.13. Please let us know if you use it and we'll put it back in RocksDB 3.14.
* Env::FileExists now returns a Status instead of a boolean
## 3.12.0 (7/2/2015) ## 3.12.0 (7/2/2015)
### New Features ### New Features

2
db/compact_files_test.cc
Unescape View File

@ -91,7 +91,7 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
// verify all compaction input files are deleted // verify all compaction input files are deleted
for (auto fname : l0_files) { for (auto fname : l0_files) {
ASSERT_TRUE(!env_->FileExists(fname)); ASSERT_EQ(Status::NotFound(), env_->FileExists(fname));
} }
delete db; delete db;
} }

15
db/db_impl.cc
Unescape View File

@ -868,7 +868,8 @@ Status DBImpl::Recover(
return s; return s;
} }
if (!env_->FileExists(CurrentFileName(dbname_))) { s = env_->FileExists(CurrentFileName(dbname_));
if (s.IsNotFound()) {
if (db_options_.create_if_missing) { if (db_options_.create_if_missing) {
s = NewDB(); s = NewDB();
is_new_db = true; is_new_db = true;
@ -879,18 +880,26 @@ Status DBImpl::Recover(
return Status::InvalidArgument( return Status::InvalidArgument(
dbname_, "does not exist (create_if_missing is false)"); dbname_, "does not exist (create_if_missing is false)");
} }
} else { } else if (s.ok()) {
if (db_options_.error_if_exists) { if (db_options_.error_if_exists) {
return Status::InvalidArgument( return Status::InvalidArgument(
dbname_, "exists (error_if_exists is true)"); dbname_, "exists (error_if_exists is true)");
} }
} else {
// Unexpected error reading file
assert(s.IsIOError());
return s;
} }
// Check for the IDENTITY file and create it if not there // Check for the IDENTITY file and create it if not there
if (!env_->FileExists(IdentityFileName(dbname_))) { s = env_->FileExists(IdentityFileName(dbname_));
if (s.IsNotFound()) {
s = SetIdentityFile(env_, dbname_); s = SetIdentityFile(env_, dbname_);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
} else if (!s.ok()) {
assert(s.IsIOError());
return s;
} }
} }

6
db/db_test.cc
Unescape View File

@ -8450,13 +8450,13 @@ TEST_F(DBTest, DeleteMovedFileAfterCompaction) {
ASSERT_EQ("0,0,2", FilesPerLevel(0)); ASSERT_EQ("0,0,2", FilesPerLevel(0));
// iterator is holding the file // iterator is holding the file
ASSERT_TRUE(env_->FileExists(dbname_ + moved_file_name)); ASSERT_OK(env_->FileExists(dbname_ + moved_file_name));
listener->SetExpectedFileName(dbname_ + moved_file_name); listener->SetExpectedFileName(dbname_ + moved_file_name);
iterator.reset(); iterator.reset();
// this file should have been compacted away // this file should have been compacted away
ASSERT_TRUE(!env_->FileExists(dbname_ + moved_file_name)); ASSERT_EQ(Status::NotFound(), env_->FileExists(dbname_ + moved_file_name));
listener->VerifyMatchedCount(1); listener->VerifyMatchedCount(1);
} }
} }
@ -8703,7 +8703,7 @@ TEST_F(DBTest, DeleteObsoleteFilesPendingOutputs) {
ASSERT_EQ(metadata.size(), 2U); ASSERT_EQ(metadata.size(), 2U);
// This file should have been deleted during last compaction // This file should have been deleted during last compaction
ASSERT_TRUE(!env_->FileExists(dbname_ + file_on_L2)); ASSERT_EQ(Status::NotFound(), env_->FileExists(dbname_ + file_on_L2));
listener->VerifyMatchedCount(1); listener->VerifyMatchedCount(1);
} }

12
db/deletefile_test.cc
Unescape View File

@ -269,11 +269,11 @@ TEST_F(DeleteFileTest, DeleteLogFiles) {
// Should not succeed because live logs are not allowed to be deleted // Should not succeed because live logs are not allowed to be deleted
std::unique_ptr<LogFile> alive_log = std::move(logfiles.back()); std::unique_ptr<LogFile> alive_log = std::move(logfiles.back());
ASSERT_EQ(alive_log->Type(), kAliveLogFile); ASSERT_EQ(alive_log->Type(), kAliveLogFile);
ASSERT_TRUE(env_->FileExists(options_.wal_dir + "/" + alive_log->PathName())); ASSERT_OK(env_->FileExists(options_.wal_dir + "/" + alive_log->PathName()));
fprintf(stdout, "Deleting alive log file %s\n", fprintf(stdout, "Deleting alive log file %s\n",
alive_log->PathName().c_str()); alive_log->PathName().c_str());
ASSERT_TRUE(!db_->DeleteFile(alive_log->PathName()).ok()); ASSERT_TRUE(!db_->DeleteFile(alive_log->PathName()).ok());
ASSERT_TRUE(env_->FileExists(options_.wal_dir + "/" + alive_log->PathName())); ASSERT_OK(env_->FileExists(options_.wal_dir + "/" + alive_log->PathName()));
logfiles.clear(); logfiles.clear();
// Call Flush to bring about a new working log file and add more keys // Call Flush to bring about a new working log file and add more keys
@ -287,13 +287,13 @@ TEST_F(DeleteFileTest, DeleteLogFiles) {
ASSERT_GT(logfiles.size(), 0UL); ASSERT_GT(logfiles.size(), 0UL);
std::unique_ptr<LogFile> archived_log = std::move(logfiles.front()); std::unique_ptr<LogFile> archived_log = std::move(logfiles.front());
ASSERT_EQ(archived_log->Type(), kArchivedLogFile); ASSERT_EQ(archived_log->Type(), kArchivedLogFile);
ASSERT_TRUE(env_->FileExists(options_.wal_dir + "/" + ASSERT_OK(
archived_log->PathName())); env_->FileExists(options_.wal_dir + "/" + archived_log->PathName()));
fprintf(stdout, "Deleting archived log file %s\n", fprintf(stdout, "Deleting archived log file %s\n",
archived_log->PathName().c_str()); archived_log->PathName().c_str());
ASSERT_OK(db_->DeleteFile(archived_log->PathName())); ASSERT_OK(db_->DeleteFile(archived_log->PathName()));
ASSERT_TRUE(!env_->FileExists(options_.wal_dir + "/" + ASSERT_EQ(Status::NotFound(), env_->FileExists(options_.wal_dir + "/" +
archived_log->PathName())); archived_log->PathName()));
CloseDB(); CloseDB();
} }

8
db/fault_injection_test.cc
Unescape View File

@ -191,10 +191,14 @@ class FaultInjectionTestEnv : public EnvWrapper {
return Status::Corruption("Not Active"); return Status::Corruption("Not Active");
} }
// Not allow overwriting files // Not allow overwriting files
if (target()->FileExists(fname)) { Status s = target()->FileExists(fname);
if (s.ok()) {
return Status::Corruption("File already exists."); return Status::Corruption("File already exists.");
} else if (!s.IsNotFound()) {
assert(s.IsIOError());
return s;
} }
Status s = target()->NewWritableFile(fname, result, soptions); s = target()->NewWritableFile(fname, result, soptions);
if (s.ok()) { if (s.ok()) {
result->reset(new TestWritableFile(fname, std::move(*result), this)); result->reset(new TestWritableFile(fname, std::move(*result), this));
// WritableFileWriter* file is opened // WritableFileWriter* file is opened

14
db/wal_manager.cc
Unescape View File

@ -59,11 +59,15 @@ Status WalManager::GetSortedWalFiles(VectorLogPtr& files) {
files.clear(); files.clear();
// list wal files in archive dir. // list wal files in archive dir.
std::string archivedir = ArchivalDirectory(db_options_.wal_dir); std::string archivedir = ArchivalDirectory(db_options_.wal_dir);
if (env_->FileExists(archivedir)) { Status exists = env_->FileExists(archivedir);
if (exists.ok()) {
s = GetSortedWalsOfType(archivedir, files, kArchivedLogFile); s = GetSortedWalsOfType(archivedir, files, kArchivedLogFile);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
} else if (!exists.IsNotFound()) {
assert(s.IsIOError());
return s;
} }
uint64_t latest_archived_log_number = 0; uint64_t latest_archived_log_number = 0;
@ -313,9 +317,9 @@ Status WalManager::GetSortedWalsOfType(const std::string& path,
// re-try in case the alive log file has been moved to archive. // re-try in case the alive log file has been moved to archive.
std::string archived_file = ArchivedLogFileName(path, number); std::string archived_file = ArchivedLogFileName(path, number);
if (!s.ok() && log_type == kAliveLogFile && if (!s.ok() && log_type == kAliveLogFile &&
env_->FileExists(archived_file)) { env_->FileExists(archived_file).ok()) {
s = env_->GetFileSize(archived_file, &size_bytes); s = env_->GetFileSize(archived_file, &size_bytes);
if (!s.ok() && !env_->FileExists(archived_file)) { if (!s.ok() && env_->FileExists(archived_file).IsNotFound()) {
// oops, the file just got deleted from archived dir! move on // oops, the file just got deleted from archived dir! move on
s = Status::OK(); s = Status::OK();
continue; continue;
@ -380,7 +384,7 @@ Status WalManager::ReadFirstRecord(const WalFileType type,
if (type == kAliveLogFile) { if (type == kAliveLogFile) {
std::string fname = LogFileName(db_options_.wal_dir, number); std::string fname = LogFileName(db_options_.wal_dir, number);
s = ReadFirstLine(fname, sequence); s = ReadFirstLine(fname, sequence);
if (env_->FileExists(fname) && !s.ok()) { if (env_->FileExists(fname).ok() && !s.ok()) {
// return any error that is not caused by non-existing file // return any error that is not caused by non-existing file
return s; return s;
} }
@ -394,7 +398,7 @@ Status WalManager::ReadFirstRecord(const WalFileType type,
// maybe the file was deleted from archive dir. If that's the case, return // maybe the file was deleted from archive dir. If that's the case, return
// Status::OK(). The caller with identify this as empty file because // Status::OK(). The caller with identify this as empty file because
// *sequence == 0 // *sequence == 0
if (!s.ok() && !env_->FileExists(archived_file)) { if (!s.ok() && env_->FileExists(archived_file).IsNotFound()) {
return Status::OK(); return Status::OK();
} }
} }

6
hdfs/env_hdfs.h
Unescape View File

@ -73,7 +73,7 @@ class HdfsEnv : public Env {
virtual Status NewDirectory(const std::string& name, virtual Status NewDirectory(const std::string& name,
std::unique_ptr<Directory>* result); std::unique_ptr<Directory>* result);
virtual bool FileExists(const std::string& fname); virtual Status FileExists(const std::string& fname);
virtual Status GetChildren(const std::string& path, virtual Status GetChildren(const std::string& path,
std::vector<std::string>* result); std::vector<std::string>* result);
@ -279,7 +279,9 @@ class HdfsEnv : public Env {
return notsup; return notsup;
} }
virtual bool FileExists(const std::string& fname) override { return false; } virtual Status FileExists(const std::string& fname) override {
return notsup;
}
virtual Status GetChildren(const std::string& path, virtual Status GetChildren(const std::string& path,
std::vector<std::string>* result) override { std::vector<std::string>* result) override {

10
include/rocksdb/env.h
Unescape View File

@ -156,8 +156,12 @@ class Env {
virtual Status NewDirectory(const std::string& name, virtual Status NewDirectory(const std::string& name,
unique_ptr<Directory>* result) = 0; unique_ptr<Directory>* result) = 0;
// Returns true iff the named file exists. // Returns OK if the named file exists.
virtual bool FileExists(const std::string& fname) = 0; // NotFound if the named file does not exist,
// the calling process does not have permission to determine
// whether this file exists, or if the path is invalid.
// IOError if an IO Error was encountered
virtual Status FileExists(const std::string& fname) = 0;
// Store in *result the names of the children of the specified directory. // Store in *result the names of the children of the specified directory.
// The names are relative to "dir". // The names are relative to "dir".
@ -764,7 +768,7 @@ class EnvWrapper : public Env {
unique_ptr<Directory>* result) override { unique_ptr<Directory>* result) override {
return target_->NewDirectory(name, result); return target_->NewDirectory(name, result);
} }
bool FileExists(const std::string& f) override { Status FileExists(const std::string& f) override {
return target_->FileExists(f); return target_->FileExists(f);
} }
Status GetChildren(const std::string& dir, Status GetChildren(const std::string& dir,

10
include/rocksdb/status.h
Unescape View File

@ -31,6 +31,8 @@ class Status {
// Copy the specified status. // Copy the specified status.
Status(const Status& s); Status(const Status& s);
void operator=(const Status& s); void operator=(const Status& s);
bool operator==(const Status& rhs) const;
bool operator!=(const Status& rhs) const;
// Return a success status. // Return a success status.
static Status OK() { return Status(); } static Status OK() { return Status(); }
@ -164,6 +166,14 @@ inline void Status::operator=(const Status& s) {
} }
} }
inline bool Status::operator==(const Status& rhs) const {
return (code_ == rhs.code_);
}
inline bool Status::operator!=(const Status& rhs) const {
return !(*this == rhs);
}
} // namespace rocksdb } // namespace rocksdb
#endif // STORAGE_ROCKSDB_INCLUDE_STATUS_H_ #endif // STORAGE_ROCKSDB_INCLUDE_STATUS_H_

5
port/win/env_win.cc
Unescape View File

@ -1663,10 +1663,11 @@ class WinEnv : public Env {
return s; return s;
} }
virtual bool FileExists(const std::string& fname) override { virtual Status FileExists(const std::string& fname) override {
// F_OK == 0 // F_OK == 0
const int F_OK_ = 0; const int F_OK_ = 0;
return _access(fname.c_str(), F_OK_) == 0; return _access(fname.c_str(), F_OK_) == 0 ? Status::OK()
: Status::NotFound();
} }
virtual Status GetChildren(const std::string& dir, virtual Status GetChildren(const std::string& dir,

3
table/cuckoo_table_reader_test.cc
Unescape View File

@ -522,7 +522,8 @@ TEST_F(CuckooReaderTest, TestReadPerformance) {
"WARNING: Not compiled with DNDEBUG. Performance tests may be slow.\n"); "WARNING: Not compiled with DNDEBUG. Performance tests may be slow.\n");
#endif #endif
for (uint64_t num : nums) { for (uint64_t num : nums) {
if (FLAGS_write || !Env::Default()->FileExists(GetFileName(num))) { if (FLAGS_write ||
Env::Default()->FileExists(GetFileName(num)).IsNotFound()) {
std::vector<std::string> all_keys; std::vector<std::string> all_keys;
GetKeys(num, &all_keys); GetKeys(num, &all_keys);
WriteFile(all_keys, num, hash_ratio); WriteFile(all_keys, num, hash_ratio);

6
util/auto_roll_logger_test.cc
Unescape View File

@ -155,9 +155,9 @@ TEST_F(AutoRollLoggerTest, RollLogFileByTime) {
InitTestDb(); InitTestDb();
// -- Test the existence of file during the server restart. // -- Test the existence of file during the server restart.
ASSERT_TRUE(!env->FileExists(kLogFile)); ASSERT_EQ(Status::NotFound(), env->FileExists(kLogFile));
AutoRollLogger logger(Env::Default(), kTestDir, "", log_size, time); AutoRollLogger logger(Env::Default(), kTestDir, "", log_size, time);
ASSERT_TRUE(env->FileExists(kLogFile)); ASSERT_OK(env->FileExists(kLogFile));
RollLogFileByTimeTest(&logger, time, kSampleMessage + ":RollLogFileByTime"); RollLogFileByTimeTest(&logger, time, kSampleMessage + ":RollLogFileByTime");
} }
@ -397,7 +397,7 @@ TEST_F(AutoRollLoggerTest, LogFileExistence) {
options.max_log_file_size = 100 * 1024 * 1024; options.max_log_file_size = 100 * 1024 * 1024;
options.create_if_missing = true; options.create_if_missing = true;
ASSERT_OK(rocksdb::DB::Open(options, kTestDir, &db)); ASSERT_OK(rocksdb::DB::Open(options, kTestDir, &db));
ASSERT_TRUE(env->FileExists(kLogFile)); ASSERT_OK(env->FileExists(kLogFile));
delete db; delete db;
} }

12
util/env_hdfs.cc
Unescape View File

@ -448,20 +448,18 @@ Status HdfsEnv::NewDirectory(const std::string& name,
} }
} }
bool HdfsEnv::FileExists(const std::string& fname) { Status HdfsEnv::FileExists(const std::string& fname) {
int value = hdfsExists(fileSys_, fname.c_str()); int value = hdfsExists(fileSys_, fname.c_str());
switch (value) { switch (value) {
case HDFS_EXISTS: case HDFS_EXISTS:
return true; return Status::OK();
case HDFS_DOESNT_EXIST: case HDFS_DOESNT_EXIST:
return false; return Status::NotFound();
default: // anything else should be an error default: // anything else should be an error
Log(InfoLogLevel::FATAL_LEVEL, Log(InfoLogLevel::FATAL_LEVEL,
mylog, "FileExists hdfsExists call failed"); mylog, "FileExists hdfsExists call failed");
throw HdfsFatalException("hdfsExists call failed with error " + return Status::IOError("hdfsExists call failed with error " +
ToString(value) + " on path " + fname + ToString(value) + " on path " + fname + ".\n");
".\n");
} }
} }

28
util/env_posix.cc
Unescape View File

@ -40,6 +40,7 @@
#include "util/posix_logger.h" #include "util/posix_logger.h"
#include "util/random.h" #include "util/random.h"
#include "util/iostats_context_imp.h" #include "util/iostats_context_imp.h"
#include "util/string_util.h"
#include "util/sync_point.h" #include "util/sync_point.h"
#include "util/thread_status_updater.h" #include "util/thread_status_updater.h"
#include "util/thread_status_util.h" #include "util/thread_status_util.h"
@ -1055,8 +1056,25 @@ class PosixEnv : public Env {
return Status::OK(); return Status::OK();
} }
virtual bool FileExists(const std::string& fname) override { virtual Status FileExists(const std::string& fname) override {
return access(fname.c_str(), F_OK) == 0; int result = access(fname.c_str(), F_OK);
if (result == 0) {
return Status::OK();
}
switch (errno) {
case EACCES:
case ELOOP:
case ENAMETOOLONG:
case ENOENT:
case ENOTDIR:
return Status::NotFound();
default:
assert(result == EIO || result == ENOMEM);
return Status::IOError("Unexpected error(" + ToString(result) +
") accessing file `" + fname + "' ");
}
} }
virtual Status GetChildren(const std::string& dir, virtual Status GetChildren(const std::string& dir,
@ -1772,9 +1790,11 @@ void PosixEnv::WaitForJoin() {
std::string Env::GenerateUniqueId() { std::string Env::GenerateUniqueId() {
std::string uuid_file = "/proc/sys/kernel/random/uuid"; std::string uuid_file = "/proc/sys/kernel/random/uuid";
if (FileExists(uuid_file)) {
Status s = FileExists(uuid_file);
if (s.ok()) {
std::string uuid; std::string uuid;
Status s = ReadFileToString(this, uuid_file, &uuid); s = ReadFileToString(this, uuid_file, &uuid);
if (s.ok()) { if (s.ok()) {
return uuid; return uuid;
} }

8
util/memenv.cc
Unescape View File

@ -308,10 +308,14 @@ class InMemoryEnv : public EnvWrapper {
return Status::OK(); return Status::OK();
} }
virtual bool FileExists(const std::string& fname) override { virtual Status FileExists(const std::string& fname) override {
std::string nfname = NormalizeFileName(fname); std::string nfname = NormalizeFileName(fname);
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
return file_map_.find(nfname) != file_map_.end(); if (file_map_.find(nfname) != file_map_.end()) {
return Status::OK();
} else {
return Status::NotFound();
}
} }
virtual Status GetChildren(const std::string& dir, virtual Status GetChildren(const std::string& dir,

10
util/memenv_test.cc
Unescape View File

@ -35,7 +35,7 @@ TEST_F(MemEnvTest, Basics) {
ASSERT_OK(env_->CreateDir("/dir")); ASSERT_OK(env_->CreateDir("/dir"));
// Check that the directory is empty. // Check that the directory is empty.
ASSERT_TRUE(!env_->FileExists("/dir/non_existent")); ASSERT_EQ(Status::NotFound(), env_->FileExists("/dir/non_existent"));
ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok()); ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok());
ASSERT_OK(env_->GetChildren("/dir", &children)); ASSERT_OK(env_->GetChildren("/dir", &children));
ASSERT_EQ(0U, children.size()); ASSERT_EQ(0U, children.size());
@ -45,7 +45,7 @@ TEST_F(MemEnvTest, Basics) {
writable_file.reset(); writable_file.reset();
// Check that the file exists. // Check that the file exists.
ASSERT_TRUE(env_->FileExists("/dir/f")); ASSERT_OK(env_->FileExists("/dir/f"));
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
ASSERT_EQ(0U, file_size); ASSERT_EQ(0U, file_size);
ASSERT_OK(env_->GetChildren("/dir", &children)); ASSERT_OK(env_->GetChildren("/dir", &children));
@ -64,8 +64,8 @@ TEST_F(MemEnvTest, Basics) {
// Check that renaming works. // Check that renaming works.
ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok());
ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g")); ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g"));
ASSERT_TRUE(!env_->FileExists("/dir/f")); ASSERT_EQ(Status::NotFound(), env_->FileExists("/dir/f"));
ASSERT_TRUE(env_->FileExists("/dir/g")); ASSERT_OK(env_->FileExists("/dir/g"));
ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); ASSERT_OK(env_->GetFileSize("/dir/g", &file_size));
ASSERT_EQ(3U, file_size); ASSERT_EQ(3U, file_size);
@ -82,7 +82,7 @@ TEST_F(MemEnvTest, Basics) {
// Check that deleting works. // Check that deleting works.
ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok()); ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok());
ASSERT_OK(env_->DeleteFile("/dir/g")); ASSERT_OK(env_->DeleteFile("/dir/g"));
ASSERT_TRUE(!env_->FileExists("/dir/g")); ASSERT_EQ(Status::NotFound(), env_->FileExists("/dir/g"));
ASSERT_OK(env_->GetChildren("/dir", &children)); ASSERT_OK(env_->GetChildren("/dir", &children));
ASSERT_EQ(0U, children.size()); ASSERT_EQ(0U, children.size());
ASSERT_OK(env_->DeleteDir("/dir")); ASSERT_OK(env_->DeleteDir("/dir"));

8
util/mock_env.cc
Unescape View File

@ -468,12 +468,12 @@ Status MockEnv::NewDirectory(const std::string& name,
return Status::OK(); return Status::OK();
} }
bool MockEnv::FileExists(const std::string& fname) { Status MockEnv::FileExists(const std::string& fname) {
auto fn = NormalizePath(fname); auto fn = NormalizePath(fname);
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
if (file_map_.find(fn) != file_map_.end()) { if (file_map_.find(fn) != file_map_.end()) {
// File exists // File exists
return true; return Status::OK();
} }
// Now also check if fn exists as a dir // Now also check if fn exists as a dir
for (const auto& iter : file_map_) { for (const auto& iter : file_map_) {
@ -481,10 +481,10 @@ bool MockEnv::FileExists(const std::string& fname) {
if (filename.size() >= fn.size() + 1 && if (filename.size() >= fn.size() + 1 &&
filename[fn.size()] == '/' && filename[fn.size()] == '/' &&
Slice(filename).starts_with(Slice(fn))) { Slice(filename).starts_with(Slice(fn))) {
return true; return Status::OK();
} }
} }
return false; return Status::NotFound();
} }
Status MockEnv::GetChildren(const std::string& dir, Status MockEnv::GetChildren(const std::string& dir,

2
util/mock_env.h
Unescape View File

@ -46,7 +46,7 @@ class MockEnv : public EnvWrapper {
virtual Status NewDirectory(const std::string& name, virtual Status NewDirectory(const std::string& name,
unique_ptr<Directory>* result) override; unique_ptr<Directory>* result) override;
virtual bool FileExists(const std::string& fname) override; virtual Status FileExists(const std::string& fname) override;
virtual Status GetChildren(const std::string& dir, virtual Status GetChildren(const std::string& dir,
std::vector<std::string>* result) override; std::vector<std::string>* result) override;

10
util/mock_env_test.cc
Unescape View File

@ -34,7 +34,7 @@ TEST_F(MockEnvTest, Basics) {
ASSERT_OK(env_->CreateDir("/dir")); ASSERT_OK(env_->CreateDir("/dir"));
// Check that the directory is empty. // Check that the directory is empty.
ASSERT_TRUE(!env_->FileExists("/dir/non_existent")); ASSERT_EQ(Status::NotFound(), env_->FileExists("/dir/non_existent"));
ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok()); ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok());
ASSERT_OK(env_->GetChildren("/dir", &children)); ASSERT_OK(env_->GetChildren("/dir", &children));
ASSERT_EQ(0U, children.size()); ASSERT_EQ(0U, children.size());
@ -44,7 +44,7 @@ TEST_F(MockEnvTest, Basics) {
writable_file.reset(); writable_file.reset();
// Check that the file exists. // Check that the file exists.
ASSERT_TRUE(env_->FileExists("/dir/f")); ASSERT_OK(env_->FileExists("/dir/f"));
ASSERT_OK(env_->GetFileSize("/dir/f", &file_size)); ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
ASSERT_EQ(0U, file_size); ASSERT_EQ(0U, file_size);
ASSERT_OK(env_->GetChildren("/dir", &children)); ASSERT_OK(env_->GetChildren("/dir", &children));
@ -63,8 +63,8 @@ TEST_F(MockEnvTest, Basics) {
// Check that renaming works. // Check that renaming works.
ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok()); ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok());
ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g")); ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g"));
ASSERT_TRUE(!env_->FileExists("/dir/f")); ASSERT_EQ(Status::NotFound(), env_->FileExists("/dir/f"));
ASSERT_TRUE(env_->FileExists("/dir/g")); ASSERT_OK(env_->FileExists("/dir/g"));
ASSERT_OK(env_->GetFileSize("/dir/g", &file_size)); ASSERT_OK(env_->GetFileSize("/dir/g", &file_size));
ASSERT_EQ(3U, file_size); ASSERT_EQ(3U, file_size);
@ -81,7 +81,7 @@ TEST_F(MockEnvTest, Basics) {
// Check that deleting works. // Check that deleting works.
ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok()); ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok());
ASSERT_OK(env_->DeleteFile("/dir/g")); ASSERT_OK(env_->DeleteFile("/dir/g"));
ASSERT_TRUE(!env_->FileExists("/dir/g")); ASSERT_EQ(Status::NotFound(), env_->FileExists("/dir/g"));
ASSERT_OK(env_->GetChildren("/dir", &children)); ASSERT_OK(env_->GetChildren("/dir", &children));
ASSERT_EQ(0U, children.size()); ASSERT_EQ(0U, children.size());
ASSERT_OK(env_->DeleteDir("/dir")); ASSERT_OK(env_->DeleteDir("/dir"));

31
utilities/backupable/backupable_db.cc
Unescape View File

@ -1117,10 +1117,12 @@ Status BackupEngineImpl::GetLatestBackupFileContents(uint32_t* latest_backup) {
*latest_backup = 0; *latest_backup = 0;
sscanf(data.data(), "%u", latest_backup); sscanf(data.data(), "%u", latest_backup);
if (backup_env_->FileExists(GetBackupMetaFile(*latest_backup)) == false) {
s = backup_env_->FileExists(GetBackupMetaFile(*latest_backup));
if (s.IsNotFound()) {
s = Status::Corruption("Latest backup file corrupted"); s = Status::Corruption("Latest backup file corrupted");
} }
return Status::OK(); return s;
} }
// this operation HAS to be atomic // this operation HAS to be atomic
@ -1283,7 +1285,21 @@ Status BackupEngineImpl::AddBackupFileWorkItem(
// true if dst_path is the same path as another live file // true if dst_path is the same path as another live file
const bool same_path = const bool same_path =
live_dst_paths.find(dst_path) != live_dst_paths.end(); live_dst_paths.find(dst_path) != live_dst_paths.end();
if (shared && (backup_env_->FileExists(dst_path) || same_path)) {
bool file_exists = false;
if (shared && !same_path) {
Status exist = backup_env_->FileExists(dst_path);
if (exist.ok()) {
file_exists = true;
} else if (exist.IsNotFound()) {
file_exists = false;
} else {
assert(s.IsIOError());
return exist;
}
}
if (shared && (same_path || file_exists)) {
need_to_copy = false; need_to_copy = false;
if (shared_checksum) { if (shared_checksum) {
Log(options_.info_log, Log(options_.info_log,
@ -1510,8 +1526,13 @@ Status BackupEngineImpl::BackupMeta::Delete(bool delete_meta) {
} }
files_.clear(); files_.clear();
// delete meta file // delete meta file
if (delete_meta && env_->FileExists(meta_filename_)) { if (delete_meta) {
s = env_->DeleteFile(meta_filename_); s = env_->FileExists(meta_filename_);
if (s.ok()) {
s = env_->DeleteFile(meta_filename_);
} else if (s.IsNotFound()) {
s = Status::OK(); // nothing to delete
}
} }
timestamp_ = 0; timestamp_ = 0;
return s; return s;

74
utilities/backupable/backupable_db_test.cc
Unescape View File

@ -584,10 +584,12 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
test_backup_env_->AssertWrittenFiles(should_have_written); test_backup_env_->AssertWrittenFiles(should_have_written);
ASSERT_OK(backup_engine_->DeleteBackup(1)); ASSERT_OK(backup_engine_->DeleteBackup(1));
ASSERT_TRUE(test_backup_env_->FileExists(backupdir_ + "/shared/00010.sst")); ASSERT_OK(test_backup_env_->FileExists(backupdir_ + "/shared/00010.sst"));
// 00011.sst was only in backup 1, should be deleted // 00011.sst was only in backup 1, should be deleted
ASSERT_FALSE(test_backup_env_->FileExists(backupdir_ + "/shared/00011.sst")); ASSERT_EQ(Status::NotFound(),
ASSERT_TRUE(test_backup_env_->FileExists(backupdir_ + "/shared/00015.sst")); test_backup_env_->FileExists(backupdir_ + "/shared/00011.sst"));
ASSERT_OK(test_backup_env_->FileExists(backupdir_ + "/shared/00015.sst"));
// MANIFEST file size should be only 100 // MANIFEST file size should be only 100
uint64_t size; uint64_t size;
@ -678,8 +680,10 @@ TEST_F(BackupableDBTest, CorruptionsTest) {
ASSERT_OK(file_manager_->WriteToFile(backupdir_ + "/LATEST_BACKUP", "5")); ASSERT_OK(file_manager_->WriteToFile(backupdir_ + "/LATEST_BACKUP", "5"));
AssertBackupConsistency(0, 0, keys_iteration * 5, keys_iteration * 6); AssertBackupConsistency(0, 0, keys_iteration * 5, keys_iteration * 6);
// assert that all 6 data is gone! // assert that all 6 data is gone!
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/6") == false); ASSERT_EQ(Status::NotFound(),
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/6") == false); file_manager_->FileExists(backupdir_ + "/meta/6"));
ASSERT_EQ(Status::NotFound(),
file_manager_->FileExists(backupdir_ + "/private/6"));
// --------- case 3. corrupted backup meta or missing backuped file ---- // --------- case 3. corrupted backup meta or missing backuped file ----
ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/meta/5", 3)); ASSERT_OK(file_manager_->CorruptFile(backupdir_ + "/meta/5", 3));
@ -705,23 +709,23 @@ TEST_F(BackupableDBTest, CorruptionsTest) {
// checksum of the backup 2 appears to be valid, this can cause checksum // checksum of the backup 2 appears to be valid, this can cause checksum
// mismatch and abort restore process // mismatch and abort restore process
ASSERT_OK(file_manager_->CorruptChecksum(backupdir_ + "/meta/2", true)); ASSERT_OK(file_manager_->CorruptChecksum(backupdir_ + "/meta/2", true));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/2")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/2"));
OpenBackupEngine(); OpenBackupEngine();
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/2")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/2"));
s = backup_engine_->RestoreDBFromBackup(2, dbname_, dbname_); s = backup_engine_->RestoreDBFromBackup(2, dbname_, dbname_);
ASSERT_TRUE(!s.ok()); ASSERT_TRUE(!s.ok());
// make sure that no corrupt backups have actually been deleted! // make sure that no corrupt backups have actually been deleted!
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/1")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/1"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/2")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/2"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/3")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/3"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/4")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/4"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/5")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/5"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/1")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/1"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/2")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/2"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/3")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/3"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/4")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/4"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/5")); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/5"));
// delete the corrupt backups and then make sure they're actually deleted // delete the corrupt backups and then make sure they're actually deleted
ASSERT_OK(backup_engine_->DeleteBackup(5)); ASSERT_OK(backup_engine_->DeleteBackup(5));
@ -729,14 +733,22 @@ TEST_F(BackupableDBTest, CorruptionsTest) {
ASSERT_OK(backup_engine_->DeleteBackup(3)); ASSERT_OK(backup_engine_->DeleteBackup(3));
ASSERT_OK(backup_engine_->DeleteBackup(2)); ASSERT_OK(backup_engine_->DeleteBackup(2));
(void)backup_engine_->GarbageCollect(); (void)backup_engine_->GarbageCollect();
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/5") == false); ASSERT_EQ(Status::NotFound(),
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/5") == false); file_manager_->FileExists(backupdir_ + "/meta/5"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/4") == false); ASSERT_EQ(Status::NotFound(),
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/4") == false); file_manager_->FileExists(backupdir_ + "/private/5"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/3") == false); ASSERT_EQ(Status::NotFound(),
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/3") == false); file_manager_->FileExists(backupdir_ + "/meta/4"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/2") == false); ASSERT_EQ(Status::NotFound(),
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/2") == false); file_manager_->FileExists(backupdir_ + "/private/4"));
ASSERT_EQ(Status::NotFound(),
file_manager_->FileExists(backupdir_ + "/meta/3"));
ASSERT_EQ(Status::NotFound(),
file_manager_->FileExists(backupdir_ + "/private/3"));
ASSERT_EQ(Status::NotFound(),
file_manager_->FileExists(backupdir_ + "/meta/2"));
ASSERT_EQ(Status::NotFound(),
file_manager_->FileExists(backupdir_ + "/private/2"));
CloseBackupEngine(); CloseBackupEngine();
AssertBackupConsistency(0, 0, keys_iteration * 1, keys_iteration * 5); AssertBackupConsistency(0, 0, keys_iteration * 1, keys_iteration * 5);
@ -772,8 +784,8 @@ TEST_F(BackupableDBTest, NoDeleteWithReadOnly) {
// assert that data from backup 5 is still here (even though LATEST_BACKUP // assert that data from backup 5 is still here (even though LATEST_BACKUP
// says 4 is latest) // says 4 is latest)
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/meta/5") == true); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/meta/5"));
ASSERT_TRUE(file_manager_->FileExists(backupdir_ + "/private/5") == true); ASSERT_OK(file_manager_->FileExists(backupdir_ + "/private/5"));
// even though 5 is here, we should only see 4 backups // even though 5 is here, we should only see 4 backups
std::vector<BackupInfo> backup_info; std::vector<BackupInfo> backup_info;
@ -997,14 +1009,14 @@ TEST_F(BackupableDBTest, DeleteTmpFiles) {
file_manager_->WriteToFile(shared_tmp, "tmp"); file_manager_->WriteToFile(shared_tmp, "tmp");
file_manager_->CreateDir(private_tmp_dir); file_manager_->CreateDir(private_tmp_dir);
file_manager_->WriteToFile(private_tmp_file, "tmp"); file_manager_->WriteToFile(private_tmp_file, "tmp");
ASSERT_TRUE(file_manager_->FileExists(private_tmp_dir)); ASSERT_OK(file_manager_->FileExists(private_tmp_dir));
OpenDBAndBackupEngine(); OpenDBAndBackupEngine();
// Need to call this explicitly to delete tmp files // Need to call this explicitly to delete tmp files
(void)backup_engine_->GarbageCollect(); (void)backup_engine_->GarbageCollect();
CloseDBAndBackupEngine(); CloseDBAndBackupEngine();
ASSERT_FALSE(file_manager_->FileExists(shared_tmp)); ASSERT_EQ(Status::NotFound(), file_manager_->FileExists(shared_tmp));
ASSERT_FALSE(file_manager_->FileExists(private_tmp_file)); ASSERT_EQ(Status::NotFound(), file_manager_->FileExists(private_tmp_file));
ASSERT_FALSE(file_manager_->FileExists(private_tmp_dir)); ASSERT_EQ(Status::NotFound(), file_manager_->FileExists(private_tmp_dir));
} }
TEST_F(BackupableDBTest, KeepLogFiles) { TEST_F(BackupableDBTest, KeepLogFiles) {

6
utilities/checkpoint/checkpoint.cc
Unescape View File

@ -64,8 +64,12 @@ Status CheckpointImpl::CreateCheckpoint(const std::string& checkpoint_dir) {
bool same_fs = true; bool same_fs = true;
VectorLogPtr live_wal_files; VectorLogPtr live_wal_files;
if (db_->GetEnv()->FileExists(checkpoint_dir)) { s = db_->GetEnv()->FileExists(checkpoint_dir);
if (s.ok()) {
return Status::InvalidArgument("Directory exists"); return Status::InvalidArgument("Directory exists");
} else if (!s.IsNotFound()) {
assert(s.IsIOError());
return s;
} }
s = db_->DisableFileDeletions(); s = db_->DisableFileDeletions();

Write Preview
Loading…
Cancel
Save