Unify DeleteFile and DeleteWalFiles

Summary:
This is to simplify rocksdb public APIs and improve the code quality.
Created an additional parameter to ParseFileName for log sub type and improved the code for deleting a wal file.
Wrote exhaustive unit-tests in delete_file_test
Unification of other redundant APIs can be taken up in a separate diff

Test Plan: Expanded delete_file test

Reviewers: dhruba, haobo, kailiu, sdong

Reviewed By: dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D13647
main
Mayank Agarwal 11 years ago
parent c17607a251
commit 56305221c4
  1. 25
      db/db_filesnapshot.cc
  2. 34
      db/db_impl.cc
  3. 1
      db/db_impl.h
  4. 2
      db/db_test.cc
  5. 38
      db/deletefile_test.cc
  6. 19
      db/filename.cc
  7. 4
      db/filename.h
  8. 13
      include/rocksdb/db.h
  9. 2
      include/rocksdb/transaction_log.h
  10. 4
      include/utilities/stackable_db.h
  11. 4
      utilities/ttl/db_ttl.cc
  12. 2
      utilities/ttl/db_ttl.h

@ -91,29 +91,4 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
return AppendSortedWalsOfType(options_.wal_dir, files, kAliveLogFile); return AppendSortedWalsOfType(options_.wal_dir, files, kAliveLogFile);
} }
Status DBImpl::DeleteWalFiles(const VectorLogPtr& files) {
Status s;
std::string archivedir = ArchivalDirectory(options_.wal_dir);
std::string files_not_deleted;
for (const auto& wal : files) {
/* Try deleting in the dir that pathname points to for the logfile.
This may fail if we try to delete a log file which was live when captured
but is archived now. Try deleting it from archive also
*/
Status st = env_->DeleteFile(options_.wal_dir + "/" + wal->PathName());
if (!st.ok()) {
if (wal->Type() == kAliveLogFile &&
env_->DeleteFile(LogFileName(archivedir, wal->LogNumber())).ok()) {
continue;
}
files_not_deleted.append(wal->PathName());
}
}
if (!files_not_deleted.empty()) {
return Status::IOError("Deleted all requested files except: " +
files_not_deleted);
}
return Status::OK();
}
} }

@ -3149,26 +3149,38 @@ inline void DBImpl::DelayLoggingAndReset() {
Status DBImpl::DeleteFile(std::string name) { Status DBImpl::DeleteFile(std::string name) {
uint64_t number; uint64_t number;
FileType type; FileType type;
if (!ParseFileName(name, &number, &type) || WalFileType log_type;
(type != kTableFile)) { if (!ParseFileName(name, &number, &type, &log_type) ||
Log(options_.info_log, (type != kTableFile && type != kLogFile)) {
"DeleteFile #%ld FAILED. Invalid file name\n", Log(options_.info_log, "DeleteFile %s failed.\n", name.c_str());
number);
return Status::InvalidArgument("Invalid file name"); return Status::InvalidArgument("Invalid file name");
} }
Status status;
if (type == kLogFile) {
// Only allow deleting archived log files
if (log_type != kArchivedLogFile) {
Log(options_.info_log, "DeleteFile %s failed.\n", name.c_str());
return Status::NotSupported("Delete only supported for archived logs");
}
status = env_->DeleteFile(options_.wal_dir + "/" + name.c_str());
if (!status.ok()) {
Log(options_.info_log, "DeleteFile %s failed.\n", name.c_str());
}
return status;
}
int level; int level;
FileMetaData metadata; FileMetaData metadata;
int maxlevel = NumberLevels(); int maxlevel = NumberLevels();
VersionEdit edit(maxlevel); VersionEdit edit(maxlevel);
DeletionState deletion_state; DeletionState deletion_state;
Status status;
{ {
MutexLock l(&mutex_); MutexLock l(&mutex_);
status = versions_->GetMetadataForFile(number, &level, &metadata); status = versions_->GetMetadataForFile(number, &level, &metadata);
if (!status.ok()) { if (!status.ok()) {
Log(options_.info_log, "DeleteFile #%lld FAILED. File not found\n", Log(options_.info_log, "DeleteFile %s failed. File not found\n",
static_cast<unsigned long long>(number)); name.c_str());
return Status::InvalidArgument("File not found"); return Status::InvalidArgument("File not found");
} }
assert((level > 0) && (level < maxlevel)); assert((level > 0) && (level < maxlevel));
@ -3176,8 +3188,7 @@ Status DBImpl::DeleteFile(std::string name) {
// If the file is being compacted no need to delete. // If the file is being compacted no need to delete.
if (metadata.being_compacted) { if (metadata.being_compacted) {
Log(options_.info_log, Log(options_.info_log,
"DeleteFile #%lld Skipped. File about to be compacted\n", "DeleteFile %s Skipped. File about to be compacted\n", name.c_str());
static_cast<unsigned long long>(number));
return Status::OK(); return Status::OK();
} }
@ -3187,8 +3198,7 @@ Status DBImpl::DeleteFile(std::string name) {
for (int i = level + 1; i < maxlevel; i++) { for (int i = level + 1; i < maxlevel; i++) {
if (versions_->NumLevelFiles(i) != 0) { if (versions_->NumLevelFiles(i) != 0) {
Log(options_.info_log, Log(options_.info_log,
"DeleteFile #%lld FAILED. File not in last level\n", "DeleteFile %s FAILED. File not in last level\n", name.c_str());
static_cast<unsigned long long>(number));
return Status::InvalidArgument("File not in last level"); return Status::InvalidArgument("File not in last level");
} }
} }

@ -73,7 +73,6 @@ class DBImpl : public DB {
uint64_t* manifest_file_size, uint64_t* manifest_file_size,
bool flush_memtable = true); bool flush_memtable = true);
virtual Status GetSortedWalFiles(VectorLogPtr& files); virtual Status GetSortedWalFiles(VectorLogPtr& files);
virtual Status DeleteWalFiles(const VectorLogPtr& files);
virtual SequenceNumber GetLatestSequenceNumber(); virtual SequenceNumber GetLatestSequenceNumber();
virtual Status GetUpdatesSince(SequenceNumber seq_number, virtual Status GetUpdatesSince(SequenceNumber seq_number,
unique_ptr<TransactionLogIterator>* iter); unique_ptr<TransactionLogIterator>* iter);

@ -4104,7 +4104,7 @@ class ModelDB: public DB {
return Status::OK(); return Status::OK();
} }
virtual Status DeleteWalFiles(const VectorLogPtr& files) { virtual Status DeleteFile(std::string name) {
return Status::OK(); return Status::OK();
} }

@ -15,6 +15,7 @@
#include "util/testharness.h" #include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/transaction_log.h"
#include <vector> #include <vector>
#include <stdlib.h> #include <stdlib.h>
#include <map> #include <map>
@ -36,6 +37,7 @@ class DeleteFileTest {
options_.write_buffer_size = 1024*1024*1000; options_.write_buffer_size = 1024*1024*1000;
options_.target_file_size_base = 1024*1024*1000; options_.target_file_size_base = 1024*1024*1000;
options_.max_bytes_for_level_base = 1024*1024*1000; options_.max_bytes_for_level_base = 1024*1024*1000;
options_.WAL_ttl_seconds = 300; // Used to test log files
dbname_ = test::TmpDir() + "/deletefile_test"; dbname_ = test::TmpDir() + "/deletefile_test";
DestroyDB(dbname_, options_); DestroyDB(dbname_, options_);
numlevels_ = 7; numlevels_ = 7;
@ -153,7 +155,6 @@ TEST(DeleteFileTest, AddKeysAndQueryLevels) {
CloseDB(); CloseDB();
} }
TEST(DeleteFileTest, DeleteFileWithIterator) { TEST(DeleteFileTest, DeleteFileWithIterator) {
CreateTwoLevels(); CreateTwoLevels();
ReadOptions options; ReadOptions options;
@ -184,6 +185,41 @@ TEST(DeleteFileTest, DeleteFileWithIterator) {
delete it; delete it;
CloseDB(); CloseDB();
} }
TEST(DeleteFileTest, DeleteLogFiles) {
AddKeys(10, 0);
VectorLogPtr logfiles;
db_->GetSortedWalFiles(logfiles);
ASSERT_GT(logfiles.size(), 0UL);
// Take the last log file which is expected to be alive and try to delete it
// Should not succeed because live logs are not allowed to be deleted
std::unique_ptr<LogFile> alive_log = std::move(logfiles.back());
ASSERT_EQ(alive_log->Type(), kAliveLogFile);
ASSERT_TRUE(env_->FileExists(dbname_ + "/" + alive_log->PathName()));
fprintf(stdout, "Deleting alive log file %s\n",
alive_log->PathName().c_str());
ASSERT_TRUE(!db_->DeleteFile(alive_log->PathName()).ok());
ASSERT_TRUE(env_->FileExists(dbname_ + "/" + alive_log->PathName()));
logfiles.clear();
// Call Flush to bring about a new working log file and add more keys
// Call Flush again to flush out memtable and move alive log to archived log
// and try to delete the archived log file
FlushOptions fopts;
db_->Flush(fopts);
AddKeys(10, 0);
db_->Flush(fopts);
db_->GetSortedWalFiles(logfiles);
ASSERT_GT(logfiles.size(), 0UL);
std::unique_ptr<LogFile> archived_log = std::move(logfiles.front());
ASSERT_EQ(archived_log->Type(), kArchivedLogFile);
ASSERT_TRUE(env_->FileExists(dbname_ + "/" + archived_log->PathName()));
fprintf(stdout, "Deleting archived log file %s\n",
archived_log->PathName().c_str());
ASSERT_OK(db_->DeleteFile(archived_log->PathName()));
ASSERT_TRUE(!env_->FileExists(dbname_ + "/" + archived_log->PathName()));
}
} //namespace rocksdb } //namespace rocksdb
int main(int argc, char** argv) { int main(int argc, char** argv) {

@ -143,7 +143,8 @@ std::string IdentityFileName(const std::string& dbname) {
// Disregards / at the beginning // Disregards / at the beginning
bool ParseFileName(const std::string& fname, bool ParseFileName(const std::string& fname,
uint64_t* number, uint64_t* number,
FileType* type) { FileType* type,
WalFileType* log_type) {
Slice rest(fname); Slice rest(fname);
if (fname.length() > 1 && fname[0] == '/') { if (fname.length() > 1 && fname[0] == '/') {
rest.remove_prefix(1); rest.remove_prefix(1);
@ -194,6 +195,17 @@ bool ParseFileName(const std::string& fname,
} else { } else {
// Avoid strtoull() to keep filename format independent of the // Avoid strtoull() to keep filename format independent of the
// current locale // current locale
bool archive_dir_found = false;
if (rest.starts_with(ARCHIVAL_DIR)) {
if (rest.size() <= ARCHIVAL_DIR.size()) {
return false;
}
rest.remove_prefix(ARCHIVAL_DIR.size() + 1); // Add 1 to remove / also
if (log_type) {
*log_type = kArchivedLogFile;
}
archive_dir_found = true;
}
uint64_t num; uint64_t num;
if (!ConsumeDecimalNumber(&rest, &num)) { if (!ConsumeDecimalNumber(&rest, &num)) {
return false; return false;
@ -201,6 +213,11 @@ bool ParseFileName(const std::string& fname,
Slice suffix = rest; Slice suffix = rest;
if (suffix == Slice(".log")) { if (suffix == Slice(".log")) {
*type = kLogFile; *type = kLogFile;
if (log_type && !archive_dir_found) {
*log_type = kAliveLogFile;
}
} else if (archive_dir_found) {
return false; // Archive dir can contain only log files
} else if (suffix == Slice(".sst")) { } else if (suffix == Slice(".sst")) {
*type = kTableFile; *type = kTableFile;
} else if (suffix == Slice(".dbtmp")) { } else if (suffix == Slice(".dbtmp")) {

@ -14,6 +14,7 @@
#include <string> #include <string>
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/transaction_log.h"
#include "port/port.h" #include "port/port.h"
namespace rocksdb { namespace rocksdb {
@ -93,7 +94,8 @@ extern std::string IdentityFileName(const std::string& dbname);
// filename was successfully parsed, returns true. Else return false. // filename was successfully parsed, returns true. Else return false.
extern bool ParseFileName(const std::string& filename, extern bool ParseFileName(const std::string& filename,
uint64_t* number, uint64_t* number,
FileType* type); FileType* type,
WalFileType* log_type = nullptr);
// Make the CURRENT file point to the descriptor file with the // Make the CURRENT file point to the descriptor file with the
// specified number. // specified number.

@ -257,10 +257,6 @@ class DB {
// Retrieve the sorted list of all wal files with earliest file first // Retrieve the sorted list of all wal files with earliest file first
virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0; virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0;
// Delete wal files in files. These can be either live or archived.
// Returns Status::OK if all files could be deleted, otherwise Status::IOError
virtual Status DeleteWalFiles(const VectorLogPtr& files) = 0;
// The sequence number of the most recent transaction. // The sequence number of the most recent transaction.
virtual SequenceNumber GetLatestSequenceNumber() = 0; virtual SequenceNumber GetLatestSequenceNumber() = 0;
@ -278,11 +274,10 @@ class DB {
virtual Status GetUpdatesSince(SequenceNumber seq_number, virtual Status GetUpdatesSince(SequenceNumber seq_number,
unique_ptr<TransactionLogIterator>* iter) = 0; unique_ptr<TransactionLogIterator>* iter) = 0;
// Delete the file name from the db directory and update the internal // Delete the file name from the db directory and update the internal state to
// state to reflect that. // reflect that. Supports deletion of sst and log files only. 'name' must be
virtual Status DeleteFile(std::string name) { // path relative to the db directory. eg. 000001.sst, /archive/000003.log
return Status::OK(); virtual Status DeleteFile(std::string name) = 0;
}
// Returns a list of all table files with their level, start key // Returns a list of all table files with their level, start key
// and end key // and end key

@ -5,6 +5,8 @@
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/types.h" #include "rocksdb/types.h"
#include "rocksdb/write_batch.h" #include "rocksdb/write_batch.h"
#include <memory>
#include <vector>
namespace rocksdb { namespace rocksdb {

@ -144,8 +144,8 @@ class StackableDB : public DB {
return sdb_->GetSortedWalFiles(files); return sdb_->GetSortedWalFiles(files);
} }
virtual Status DeleteWalFiles(const VectorLogPtr& files) override { virtual Status DeleteFile(std::string name) override {
return sdb_->DeleteWalFiles(files); return sdb_->DeleteFile(name);
} }
virtual Status GetUpdatesSince(SequenceNumber seq_number, virtual Status GetUpdatesSince(SequenceNumber seq_number,

@ -280,8 +280,8 @@ Status DBWithTTL::GetSortedWalFiles(VectorLogPtr& files) {
return db_->GetSortedWalFiles(files); return db_->GetSortedWalFiles(files);
} }
Status DBWithTTL::DeleteWalFiles(const VectorLogPtr& files){ Status DBWithTTL::DeleteFile(std::string name) {
return db_->DeleteWalFiles(files); return db_->DeleteFile(name);
} }
Status DBWithTTL::GetUpdatesSince( Status DBWithTTL::GetUpdatesSince(

@ -78,7 +78,7 @@ class DBWithTTL : public StackableDB {
virtual Status GetSortedWalFiles(VectorLogPtr& files); virtual Status GetSortedWalFiles(VectorLogPtr& files);
virtual Status DeleteWalFiles(const VectorLogPtr& files); virtual Status DeleteFile(std::string name);
virtual SequenceNumber GetLatestSequenceNumber(); virtual SequenceNumber GetLatestSequenceNumber();

Loading…
Cancel
Save