NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Small changes in Deleting obsolete files

Browse Source 
Summary:
@haobo's suggestions from https://reviews.facebook.net/D13827

Renaming some variables, deprecating purge_log_after_flush, changing for loop into auto for loop.

I have not implemented deleting objects outside of mutex yet because it would require a big code change - we would delete object in db_impl, which currently does not know anything about object because it's defined in version_edit.h (FileMetaData). We should do it at some point, though.

Test Plan: Ran deletefile_test

Reviewers: haobo

Reviewed By: haobo

CC: leveldb, haobo

Differential Revision: https://reviews.facebook.net/D14025
main
Igor Canadi 11 years ago
parent dad425562f
commit 9bc4a26f56
6 changed files with 51 additions and 56 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 61
      db/db_impl.cc
  2. 17
      db/db_impl.h
  3. 19
      db/version_set.cc
  4. 2
      db/version_set.h
  5. 5
      include/rocksdb/options.h
  6. 3
      util/options.cc

61
db/db_impl.cc
Unescape View File

@ -435,8 +435,8 @@ void DBImpl::MaybeDumpStats() {
} }
} }
// Returns the list of live files in 'sstlive' and the list // Returns the list of live files in 'sst_live' and the list
// of all files in the filesystem in 'allfiles'. // of all files in the filesystem in 'all_files'.
void DBImpl::FindObsoleteFiles(DeletionState& deletion_state, bool force) { void DBImpl::FindObsoleteFiles(DeletionState& deletion_state, bool force) {
mutex_.AssertHeld(); mutex_.AssertHeld();
@ -464,19 +464,19 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state, bool force) {
// Make a list of all of the live files; set is slow, should not // Make a list of all of the live files; set is slow, should not
// be used. // be used.
deletion_state.sstlive.assign(pending_outputs_.begin(), deletion_state.sst_live.assign(pending_outputs_.begin(),
pending_outputs_.end()); pending_outputs_.end());
versions_->AddLiveFiles(&deletion_state.sstlive); versions_->AddLiveFiles(&deletion_state.sst_live);
// set of all files in the directory // set of all files in the directory
env_->GetChildren(dbname_, &deletion_state.allfiles); // Ignore errors env_->GetChildren(dbname_, &deletion_state.all_files); // Ignore errors
//Add log files in wal_dir //Add log files in wal_dir
if (options_.wal_dir != dbname_) { if (options_.wal_dir != dbname_) {
std::vector<std::string> log_files; std::vector<std::string> log_files;
env_->GetChildren(options_.wal_dir, &log_files); // Ignore errors env_->GetChildren(options_.wal_dir, &log_files); // Ignore errors
deletion_state.allfiles.insert( deletion_state.all_files.insert(
deletion_state.allfiles.end(), deletion_state.all_files.end(),
log_files.begin(), log_files.begin(),
log_files.end() log_files.end()
); );
@ -485,7 +485,7 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state, bool force) {
// Diffs the files listed in filenames and those that do not // Diffs the files listed in filenames and those that do not
// belong to live files are posibly removed. Also, removes all the // belong to live files are posibly removed. Also, removes all the
// files in sstdeletefiles and logdeletefiles. // files in sst_delete_files and log_delete_files.
// It is not necessary to hold the mutex when invoking this method. // It is not necessary to hold the mutex when invoking this method.
void DBImpl::PurgeObsoleteFiles(DeletionState& state) { void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
// if deletion is disabled, do nothing // if deletion is disabled, do nothing
@ -499,23 +499,26 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
// Now, convert live list to an unordered set, WITHOUT mutex held; // Now, convert live list to an unordered set, WITHOUT mutex held;
// set is slow. // set is slow.
std::unordered_set<uint64_t> live_set(state.sstlive.begin(), std::unordered_set<uint64_t> live_set(state.sst_live.begin(),
state.sstlive.end()); state.sst_live.end());
state.allfiles.reserve(state.allfiles.size() + state.sstdeletefiles.size()); state.all_files.reserve(state.all_files.size() +
for (auto filenum : state.sstdeletefiles) { state.sst_delete_files.size());
state.allfiles.push_back(TableFileName("", filenum)); for (auto file : state.sst_delete_files) {
state.all_files.push_back(TableFileName("", file->number));
delete file;
} }
state.allfiles.reserve(state.allfiles.size() + state.logdeletefiles.size()); state.all_files.reserve(state.all_files.size() +
for (auto filenum : state.logdeletefiles) { state.log_delete_files.size());
for (auto filenum : state.log_delete_files) {
if (filenum > 0) { if (filenum > 0) {
state.allfiles.push_back(LogFileName("", filenum)); state.all_files.push_back(LogFileName("", filenum));
} }
} }
for (size_t i = 0; i < state.allfiles.size(); i++) { for (size_t i = 0; i < state.all_files.size(); i++) {
if (ParseFileName(state.allfiles[i], &number, &type)) { if (ParseFileName(state.all_files[i], &number, &type)) {
bool keep = true; bool keep = true;
switch (type) { switch (type) {
case kLogFile: case kLogFile:
@ -538,7 +541,7 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
case kInfoLogFile: case kInfoLogFile:
keep = true; keep = true;
if (number != 0) { if (number != 0) {
old_log_files.push_back(state.allfiles[i]); old_log_files.push_back(state.all_files[i]);
} }
break; break;
case kCurrentFile: case kCurrentFile:
@ -559,14 +562,14 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
Status st; Status st;
if (type == kLogFile && (options_.WAL_ttl_seconds > 0 || if (type == kLogFile && (options_.WAL_ttl_seconds > 0 ||
options_.WAL_size_limit_MB > 0)) { options_.WAL_size_limit_MB > 0)) {
st = env_->RenameFile(dbname_ + "/" + state.allfiles[i], st = env_->RenameFile(dbname_ + "/" + state.all_files[i],
ArchivedLogFileName(options_.wal_dir, ArchivedLogFileName(options_.wal_dir,
number)); number));
if (!st.ok()) { if (!st.ok()) {
Log(options_.info_log, "RenameFile logfile #%lu FAILED", number); Log(options_.info_log, "RenameFile logfile #%lu FAILED", number);
} }
} else { } else {
st = env_->DeleteFile(dbname_ + "/" + state.allfiles[i]); st = env_->DeleteFile(dbname_ + "/" + state.all_files[i]);
if (!st.ok()) { if (!st.ok()) {
Log(options_.info_log, "Delete type=%d #%lu FAILED\n", Log(options_.info_log, "Delete type=%d #%lu FAILED\n",
int(type), number); int(type), number);
@ -1132,12 +1135,12 @@ Status DBImpl::FlushMemTableToOutputFile(bool* madeProgress,
MaybeScheduleLogDBDeployStats(); MaybeScheduleLogDBDeployStats();
if (options_.purge_log_after_memtable_flush && if (!disable_delete_obsolete_files_) {
!disable_delete_obsolete_files_) {
// add to deletion state // add to deletion state
deletion_state.logdeletefiles.insert(deletion_state.logdeletefiles.end(), deletion_state.log_delete_files.insert(
logs_to_delete.begin(), deletion_state.log_delete_files.end(),
logs_to_delete.end()); logs_to_delete.begin(),
logs_to_delete.end());
} }
} }
return s; return s;
@ -1776,7 +1779,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
CleanupCompaction(compact, status); CleanupCompaction(compact, status);
versions_->ReleaseCompactionFiles(c.get(), status); versions_->ReleaseCompactionFiles(c.get(), status);
c->ReleaseInputs(); c->ReleaseInputs();
versions_->GetAndFreeObsoleteFiles(&deletion_state.sstdeletefiles); versions_->GetObsoleteFiles(&deletion_state.sst_delete_files);
*madeProgress = true; *madeProgress = true;
} }
c.reset(); c.reset();
@ -3366,7 +3369,7 @@ Status DBImpl::DeleteFile(std::string name) {
edit.DeleteFile(level, number); edit.DeleteFile(level, number);
status = versions_->LogAndApply(&edit, &mutex_); status = versions_->LogAndApply(&edit, &mutex_);
if (status.ok()) { if (status.ok()) {
versions_->GetAndFreeObsoleteFiles(&deletion_state.sstdeletefiles); versions_->GetObsoleteFiles(&deletion_state.sst_delete_files);
} }
FindObsoleteFiles(deletion_state, false); FindObsoleteFiles(deletion_state, false);
} // lock released here } // lock released here

17
db/db_impl.h
Unescape View File

@ -14,6 +14,7 @@
#include "db/dbformat.h" #include "db/dbformat.h"
#include "db/log_writer.h" #include "db/log_writer.h"
#include "db/snapshot.h" #include "db/snapshot.h"
#include "db/version_edit.h"
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/memtablerep.h" #include "rocksdb/memtablerep.h"
@ -159,22 +160,24 @@ class DBImpl : public DB {
struct DeletionState { struct DeletionState {
inline bool HaveSomethingToDelete() const { inline bool HaveSomethingToDelete() const {
return allfiles.size() || sstdeletefiles.size() || logdeletefiles.size(); return all_files.size() ||
sst_delete_files.size() ||
log_delete_files.size();
} }
// a list of all files that we'll consider deleting // a list of all files that we'll consider deleting
// (every once in a while this is filled up with all files // (every once in a while this is filled up with all files
// in the DB directory) // in the DB directory)
std::vector<std::string> allfiles; std::vector<std::string> all_files;
// the list of all live sst files that cannot be deleted // the list of all live sst files that cannot be deleted
std::vector<uint64_t> sstlive; std::vector<uint64_t> sst_live;
// a list of sst files that we need to delete // a list of sst files that we need to delete
std::vector<uint64_t> sstdeletefiles; std::vector<FileMetaData*> sst_delete_files;
// a list of log files that we need to delete // a list of log files that we need to delete
std::vector<uint64_t> logdeletefiles; std::vector<uint64_t> log_delete_files;
// the current manifest_file_number, log_number and prev_log_number // the current manifest_file_number, log_number and prev_log_number
// that corresponds to the set of files in 'live'. // that corresponds to the set of files in 'live'.
@ -241,7 +244,7 @@ class DBImpl : public DB {
void ReleaseCompactionUnusedFileNumbers(CompactionState* compact); void ReleaseCompactionUnusedFileNumbers(CompactionState* compact);
// Returns the list of live files in 'live' and the list // Returns the list of live files in 'live' and the list
// of all files in the filesystem in 'allfiles'. // of all files in the filesystem in 'all_files'.
// If force == false and the last call was less than // If force == false and the last call was less than
// options_.delete_obsolete_files_period_micros microseconds ago, // options_.delete_obsolete_files_period_micros microseconds ago,
// it will not fill up the deletion_state // it will not fill up the deletion_state
@ -249,7 +252,7 @@ class DBImpl : public DB {
// Diffs the files listed in filenames and those that do not // Diffs the files listed in filenames and those that do not
// belong to live files are posibly removed. Also, removes all the // belong to live files are posibly removed. Also, removes all the
// files in sstdeletefiles and logdeletefiles. // files in sst_delete_files and log_delete_files.
// It is not necessary to hold the mutex when invoking this method. // It is not necessary to hold the mutex when invoking this method.
void PurgeObsoleteFiles(DeletionState& deletion_state); void PurgeObsoleteFiles(DeletionState& deletion_state);

19
db/version_set.cc
Unescape View File

@ -1161,7 +1161,10 @@ VersionSet::VersionSet(const std::string& dbname,
VersionSet::~VersionSet() { VersionSet::~VersionSet() {
current_->Unref(); current_->Unref();
assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty
GetAndFreeObsoleteFiles(nullptr); for (auto file : obsolete_files_) {
delete file;
}
obsolete_files_.clear();
delete[] compact_pointer_; delete[] compact_pointer_;
delete[] max_file_size_; delete[] max_file_size_;
delete[] level_max_bytes_; delete[] level_max_bytes_;
@ -2861,16 +2864,10 @@ void VersionSet::GetLiveFilesMetaData(
} }
} }
void VersionSet::GetAndFreeObsoleteFiles(std::vector<uint64_t>* files) { void VersionSet::GetObsoleteFiles(std::vector<FileMetaData*>* files) {
if (files != nullptr) { files->insert(files->end(),
files->reserve(files->size() + obsolete_files_.size()); obsolete_files_.begin(),
} obsolete_files_.end());
for (size_t i = 0; i < obsolete_files_.size(); i++) {
if (files != nullptr) {
files->push_back(obsolete_files_[i]->number);
}
delete obsolete_files_[i];
}
obsolete_files_.clear(); obsolete_files_.clear();
} }

2
db/version_set.h
Unescape View File

@ -431,7 +431,7 @@ class VersionSet {
void GetLiveFilesMetaData( void GetLiveFilesMetaData(
std::vector<LiveFileMetaData> *metadata); std::vector<LiveFileMetaData> *metadata);
void GetAndFreeObsoleteFiles(std::vector<uint64_t>* files); void GetObsoleteFiles(std::vector<FileMetaData*>* files);
private: private:
class Builder; class Builder;

5
include/rocksdb/options.h
Unescape View File

@ -609,11 +609,6 @@ struct Options {
// Default: a factory that doesn't provide any object // Default: a factory that doesn't provide any object
std::shared_ptr<CompactionFilterFactory> compaction_filter_factory; std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
// Remove the log file immediately after the corresponding memtable is flushed
// to data file.
// Default: true
bool purge_log_after_memtable_flush;
// This option allows user to to collect their own interested statistics of // This option allows user to to collect their own interested statistics of
// the tables. // the tables.
// Default: emtpy vector -- no user-defined statistics collection will be // Default: emtpy vector -- no user-defined statistics collection will be

3
util/options.cc
Unescape View File

@ -101,7 +101,6 @@ Options::Options()
compaction_filter_factory( compaction_filter_factory(
std::shared_ptr<CompactionFilterFactory>( std::shared_ptr<CompactionFilterFactory>(
new DefaultCompactionFilterFactory())), new DefaultCompactionFilterFactory())),
purge_log_after_memtable_flush(true),
inplace_update_support(false), inplace_update_support(false),
inplace_update_num_locks(10000) { inplace_update_num_locks(10000) {
assert(memtable_factory.get() != nullptr); assert(memtable_factory.get() != nullptr);
@ -280,8 +279,6 @@ Options::Dump(Logger* log) const
Log(log,"Options.compaction_options_universal." Log(log,"Options.compaction_options_universal."
"max_size_amplification_percent: %u", "max_size_amplification_percent: %u",
compaction_options_universal.max_size_amplification_percent); compaction_options_universal.max_size_amplification_percent);
Log(log," Options.purge_log_after_memtable_flush: %d",
purge_log_after_memtable_flush);
std::string collector_names; std::string collector_names;
for (auto collector : table_stats_collectors) { for (auto collector : table_stats_collectors) {
collector_names.append(collector->Name()); collector_names.append(collector->Name());

Write Preview
Loading…
Cancel
Save