@ -213,7 +213,9 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
bg_flush_scheduled_ ( 0 ) ,
bg_flush_scheduled_ ( 0 ) ,
manual_compaction_ ( nullptr ) ,
manual_compaction_ ( nullptr ) ,
disable_delete_obsolete_files_ ( 0 ) ,
disable_delete_obsolete_files_ ( 0 ) ,
delete_obsolete_files_last_run_ ( options . env - > NowMicros ( ) ) ,
delete_obsolete_files_next_run_ (
options . env - > NowMicros ( ) +
db_options_ . delete_obsolete_files_period_micros ) ,
last_stats_dump_time_microsec_ ( 0 ) ,
last_stats_dump_time_microsec_ ( 0 ) ,
flush_on_destroy_ ( false ) ,
flush_on_destroy_ ( false ) ,
env_options_ ( options ) ,
env_options_ ( options ) ,
@ -421,14 +423,17 @@ void DBImpl::MaybeDumpStats() {
}
}
}
}
// Returns the list of live files in 'sst_live' and the list
// If it's doing full scan:
// of all files in the filesystem in 'candidate_files'.
// * Returns the list of live files in 'full_scan_sst_live' and the list
// of all files in the filesystem in 'full_scan_candidate_files'.
// Otherwise, gets obsolete files from VersionSet.
// no_full_scan = true -- never do the full scan using GetChildren()
// no_full_scan = true -- never do the full scan using GetChildren()
// force = false -- don't force the full scan, except every
// force = false -- don't force the full scan, except every
// db_options_.delete_obsolete_files_period_micros
// db_options_.delete_obsolete_files_period_micros
// force = true -- force the full scan
// force = true -- force the full scan
void DBImpl : : FindObsoleteFiles ( JobContext * job_context , bool force ,
void DBImpl : : FindObsoleteFiles ( JobContext * job_context , bool force ,
bool no_full_scan ) {
bool no_full_scan ) {
// TODO(icanadi) clean up FindObsoleteFiles, no need to do full scans anymore
mutex_ . AssertHeld ( ) ;
mutex_ . AssertHeld ( ) ;
// if deletion is disabled, do nothing
// if deletion is disabled, do nothing
@ -445,10 +450,10 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
doing_the_full_scan = true ;
doing_the_full_scan = true ;
} else {
} else {
const uint64_t now_micros = env_ - > NowMicros ( ) ;
const uint64_t now_micros = env_ - > NowMicros ( ) ;
if ( delete_obsolete_files_last_run_ +
if ( delete_obsolete_files_next_run_ < now_micros ) {
db_options_ . delete_obsolete_files_period_micros < now_micros ) {
doing_the_full_scan = true ;
doing_the_full_scan = true ;
delete_obsolete_files_last_run_ = now_micros ;
delete_obsolete_files_next_run_ =
now_micros + db_options_ . delete_obsolete_files_period_micros ;
}
}
}
}
@ -462,13 +467,6 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
job_context - > log_number = versions_ - > MinLogNumber ( ) ;
job_context - > log_number = versions_ - > MinLogNumber ( ) ;
job_context - > prev_log_number = versions_ - > prev_log_number ( ) ;
job_context - > prev_log_number = versions_ - > prev_log_number ( ) ;
if ( ! doing_the_full_scan & & ! job_context - > HaveSomethingToDelete ( ) ) {
// avoid filling up sst_live if we're sure that we
// are not going to do the full scan and that we don't have
// anything to delete at the moment
return ;
}
// don't delete live files
// don't delete live files
if ( pending_outputs_ . size ( ) ) {
if ( pending_outputs_ . size ( ) ) {
job_context - > min_pending_output = * pending_outputs_ . begin ( ) ;
job_context - > min_pending_output = * pending_outputs_ . begin ( ) ;
@ -476,11 +474,16 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
// delete all of them
// delete all of them
job_context - > min_pending_output = std : : numeric_limits < uint64_t > : : max ( ) ;
job_context - > min_pending_output = std : : numeric_limits < uint64_t > : : max ( ) ;
}
}
versions_ - > AddLiveFiles ( & job_context - > sst_live ) ;
if ( doing_the_full_scan ) {
if ( doing_the_full_scan ) {
for ( uint32_t path_id = 0 ;
// Here we find all files in the DB directory and all the live files. In the
path_id < db_options_ . db_paths . size ( ) ; path_id + + ) {
// DeleteObsoleteFiles(), we will calculate a set difference (all_files -
// live_files) and delete all files in that difference. If we're not doing
// the full scan we don't need to get live files, because all files returned
// by GetObsoleteFiles() will be dead (and need to be deleted)
versions_ - > AddLiveFiles ( & job_context - > full_scan_sst_live ) ;
for ( uint32_t path_id = 0 ; path_id < db_options_ . db_paths . size ( ) ;
path_id + + ) {
// set of all files in the directory. We'll exclude files that are still
// set of all files in the directory. We'll exclude files that are still
// alive in the subsequent processings.
// alive in the subsequent processings.
std : : vector < std : : string > files ;
std : : vector < std : : string > files ;
@ -488,7 +491,8 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
& files ) ; // Ignore errors
& files ) ; // Ignore errors
for ( std : : string file : files ) {
for ( std : : string file : files ) {
// TODO(icanadi) clean up this mess to avoid having one-off "/" prefixes
// TODO(icanadi) clean up this mess to avoid having one-off "/" prefixes
job_context - > candidate_files . emplace_back ( " / " + file , path_id ) ;
job_context - > full_scan_candidate_files . emplace_back ( " / " + file ,
path_id ) ;
}
}
}
}
@ -497,7 +501,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
std : : vector < std : : string > log_files ;
std : : vector < std : : string > log_files ;
env_ - > GetChildren ( db_options_ . wal_dir , & log_files ) ; // Ignore errors
env_ - > GetChildren ( db_options_ . wal_dir , & log_files ) ; // Ignore errors
for ( std : : string log_file : log_files ) {
for ( std : : string log_file : log_files ) {
job_context - > candidate_files . emplace_back ( log_file , 0 ) ;
job_context - > full_scan_ candidate_files. emplace_back ( log_file , 0 ) ;
}
}
}
}
// Add info log files in db_log_dir
// Add info log files in db_log_dir
@ -506,7 +510,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
// Ignore errors
// Ignore errors
env_ - > GetChildren ( db_options_ . db_log_dir , & info_log_files ) ;
env_ - > GetChildren ( db_options_ . db_log_dir , & info_log_files ) ;
for ( std : : string log_file : info_log_files ) {
for ( std : : string log_file : info_log_files ) {
job_context - > candidate_files . emplace_back ( log_file , 0 ) ;
job_context - > full_scan_ candidate_files. emplace_back ( log_file , 0 ) ;
}
}
}
}
}
}
@ -543,11 +547,11 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
// Now, convert live list to an unordered map, WITHOUT mutex held;
// Now, convert live list to an unordered map, WITHOUT mutex held;
// set is slow.
// set is slow.
std : : unordered_map < uint64_t , const FileDescriptor * > sst_live_map ;
std : : unordered_map < uint64_t , const FileDescriptor * > sst_live_map ;
for ( const FileDescriptor & fd : state . sst_live ) {
for ( const FileDescriptor & fd : state . full_scan_ sst_live) {
sst_live_map [ fd . GetNumber ( ) ] = & fd ;
sst_live_map [ fd . GetNumber ( ) ] = & fd ;
}
}
auto candidate_files = state . candidate_files ;
auto candidate_files = state . full_scan_ candidate_files;
candidate_files . reserve ( candidate_files . size ( ) +
candidate_files . reserve ( candidate_files . size ( ) +
state . sst_delete_files . size ( ) +
state . sst_delete_files . size ( ) +
state . log_delete_files . size ( ) ) ;
state . log_delete_files . size ( ) ) ;
@ -1491,6 +1495,7 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
VersionEdit edit ;
VersionEdit edit ;
edit . SetColumnFamily ( cfd - > GetID ( ) ) ;
edit . SetColumnFamily ( cfd - > GetID ( ) ) ;
for ( const auto & f : cfd - > current ( ) - > storage_info ( ) - > LevelFiles ( level ) ) {
for ( const auto & f : cfd - > current ( ) - > storage_info ( ) - > LevelFiles ( level ) ) {
f - > moved = true ;
edit . DeleteFile ( level , f - > fd . GetNumber ( ) ) ;
edit . DeleteFile ( level , f - > fd . GetNumber ( ) ) ;
edit . AddFile ( to_level , f - > fd . GetNumber ( ) , f - > fd . GetPathId ( ) ,
edit . AddFile ( to_level , f - > fd . GetNumber ( ) , f - > fd . GetPathId ( ) ,
f - > fd . GetFileSize ( ) , f - > smallest , f - > largest ,
f - > fd . GetFileSize ( ) , f - > smallest , f - > largest ,
@ -2137,6 +2142,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
// Move file to next level
// Move file to next level
assert ( c - > num_input_files ( 0 ) = = 1 ) ;
assert ( c - > num_input_files ( 0 ) = = 1 ) ;
FileMetaData * f = c - > input ( 0 , 0 ) ;
FileMetaData * f = c - > input ( 0 , 0 ) ;
f - > moved = true ;
c - > edit ( ) - > DeleteFile ( c - > level ( ) , f - > fd . GetNumber ( ) ) ;
c - > edit ( ) - > DeleteFile ( c - > level ( ) , f - > fd . GetNumber ( ) ) ;
c - > edit ( ) - > AddFile ( c - > level ( ) + 1 , f - > fd . GetNumber ( ) , f - > fd . GetPathId ( ) ,
c - > edit ( ) - > AddFile ( c - > level ( ) + 1 , f - > fd . GetNumber ( ) , f - > fd . GetPathId ( ) ,
f - > fd . GetFileSize ( ) , f - > smallest , f - > largest ,
f - > fd . GetFileSize ( ) , f - > smallest , f - > largest ,