@ -210,29 +210,15 @@ Status CompactionJob::Run() {
ThreadStatus : : STAGE_COMPACTION_RUN ) ;
TEST_SYNC_POINT ( " CompactionJob::Run():Start " ) ;
log_buffer_ - > FlushBufferToLog ( ) ;
ColumnFamilyData * cfd = compact_ - > compaction - > column_family_data ( ) ;
auto * compaction = compact_ - > compaction ;
LogCompaction ( cfd , compaction ) ;
LogCompaction ( compaction - > column_ family_ data ( ) , compaction ) ;
int64_t imm_micros = 0 ; // Micros spent doing imm_ compactions
const uint64_t start_micros = env_ - > NowMicros ( ) ;
std : : unique_ptr < Iterator > input (
versions_ - > MakeInputIterator ( compact_ - > compaction ) ) ;
input - > SeekToFirst ( ) ;
int64_t imm_micros = 0 ; // Micros spent doing imm_ compactions
std : : unique_ptr < Iterator > input ( versions_ - > MakeInputIterator ( compaction ) ) ;
input - > SeekToFirst ( ) ;
auto status = ProcessKeyValueCompaction ( & imm_micros , input . get ( ) ) ;
if ( status . ok ( ) & &
( shutting_down_ - > load ( std : : memory_order_acquire ) | | cfd - > IsDropped ( ) ) ) {
status = Status : : ShutdownInProgress (
" Database shutdown or Column family drop during compaction " ) ;
}
if ( status . ok ( ) & & compact_ - > builder ! = nullptr ) {
status = FinishCompactionOutputFile ( input - > status ( ) ) ;
}
if ( status . ok ( ) ) {
status = input - > status ( ) ;
}
input . reset ( ) ;
if ( output_directory_ & & ! db_options_ . disableDataSync ) {
@ -331,7 +317,7 @@ Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
false /* internal key corruption is expected */ ) ;
auto compaction_filter = cfd - > ioptions ( ) - > compaction_filter ;
std : : unique_ptr < CompactionFilter > compaction_filter_from_factory = nullptr ;
if ( ! compaction_filte r) {
if ( compaction_filter = = nullpt r) {
compaction_filter_from_factory =
compact_ - > compaction - > CreateCompactionFilter ( ) ;
compaction_filter = compaction_filter_from_factory . get ( ) ;
@ -346,6 +332,9 @@ Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
StopWatchNano timer ( env_ , stats_ ! = nullptr ) ;
uint64_t total_filter_time = 0 ;
// TODO(noetzli): check whether we could check !shutting_down_->... only
// only occasionally (see diff D42687)
while ( input - > Valid ( ) & & ! shutting_down_ - > load ( std : : memory_order_acquire ) & &
! cfd - > IsDropped ( ) & & status . ok ( ) ) {
compact_ - > num_input_records + + ;
@ -360,10 +349,8 @@ Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
Slice value = input - > value ( ) ;
if ( compaction_job_stats_ ! = nullptr ) {
compaction_job_stats_ - > total_input_raw_key_bytes + =
input - > key ( ) . size ( ) ;
compaction_job_stats_ - > total_input_raw_value_bytes + =
input - > value ( ) . size ( ) ;
compaction_job_stats_ - > total_input_raw_key_bytes + = key . size ( ) ;
compaction_job_stats_ - > total_input_raw_value_bytes + = value . size ( ) ;
}
if ( compact_ - > compaction - > ShouldStopBefore ( key ) & &
@ -375,8 +362,6 @@ Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
}
// Handle key/value, add to state, etc.
bool drop = false ;
bool current_entry_is_merging = false ;
if ( ! ParseInternalKey ( key , & ikey ) ) {
// Do not hide error keys
// TODO: error key stays in db forever? Figure out the intention/rationale
@ -385,7 +370,16 @@ Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
has_current_user_key = false ;
last_sequence_for_key = kMaxSequenceNumber ;
visible_in_snapshot = kMaxSequenceNumber ;
} else {
if ( compaction_job_stats_ ! = nullptr ) {
compaction_job_stats_ - > num_corrupt_keys + + ;
}
status = WriteKeyValue ( key , value , ikey , input - > status ( ) ) ;
input - > Next ( ) ;
continue ;
}
if ( compaction_job_stats_ ! = nullptr & & ikey . type = = kTypeDeletion ) {
compaction_job_stats_ - > num_input_deletion_records + + ;
}
@ -449,8 +443,8 @@ Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
// Hidden by an newer entry for same user key
// TODO: why not > ?
assert ( last_sequence_for_key > = ikey . sequence ) ;
drop = true ; // (A)
+ + key_drop_newer_entry ;
input - > Next ( ) ; // (A)
} else if ( ikey . type = = kTypeDeletion & &
ikey . sequence < = earliest_snapshot_ & &
compact_ - > compaction - > KeyNotExistsBeyondOutputLevel (
@ -462,8 +456,8 @@ Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
// smaller sequence numbers will be dropped in the next
// few iterations of this loop (by rule (A) above).
// Therefore this deletion marker is obsolete and can be dropped.
drop = true ;
+ + key_drop_obsolete ;
input - > Next ( ) ;
} else if ( ikey . type = = kTypeMerge ) {
if ( ! merge . HasOperator ( ) ) {
LogToBuffer ( log_buffer_ , " Options::merge_operator is null. " ) ;
@ -478,84 +472,59 @@ Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
// logic could also be nicely re-used for memtable flush purge
// optimization in BuildTable.
merge . MergeUntil ( input , prev_snapshot , bottommost_level_ ,
db_options_ . statistics . get ( ) , nullptr , env_ ) ;
db_options_ . statistics . get ( ) , env_ ) ;
current_entry_is_merging = true ;
if ( merge . IsSuccess ( ) ) {
// Successfully found Put/Delete/(end-of-key-range) while merging
// Get the merge result
key = merge . key ( ) ;
ParseInternalKey ( key , & ikey ) ;
value = merge . value ( ) ;
status = WriteKeyValue ( key , merge . value ( ) , ikey , input - > status ( ) ) ;
} else {
// Did not find a Put/Delete/(end-of-key-range) while merging
// We now have some stack of merge operands to write out.
// NOTE: key,value, and ikey are now referring to old entries.
// These will be correctly set below.
assert ( ! merge . keys ( ) . empty ( ) ) ;
assert ( merge . keys ( ) . size ( ) = = merge . values ( ) . size ( ) ) ;
// Hack to make sure last_sequence_for_key is correct
ParseInternalKey ( merge . keys ( ) . front ( ) , & ikey ) ;
}
}
last_sequence_for_key = ikey . sequence ;
visible_in_snapshot = visible ;
}
if ( ! drop ) {
// We may write a single key (e.g.: for Put/Delete or successful merge).
// Or we may instead have to write a sequence/list of keys.
// We have to write a sequence iff we have an unsuccessful merge
if ( current_entry_is_merging & & ! merge . IsSuccess ( ) ) {
const auto & keys = merge . keys ( ) ;
const auto & values = merge . values ( ) ;
std : : deque < std : : string > : : const_reverse_iterator key_iter =
keys . rbegin ( ) ; // The back (*rbegin()) is the first key
std : : deque < std : : string > : : const_reverse_iterator value_iter =
values . rbegin ( ) ;
assert ( ! keys . empty ( ) ) ;
assert ( keys . size ( ) = = values . size ( ) ) ;
// We have a list of keys to write, write all keys in the list.
for ( auto key_iter = keys . rbegin ( ) , value_iter = values . rbegin ( ) ;
! status . ok ( ) | | key_iter ! = keys . rend ( ) ;
key_iter + + , value_iter + + ) {
key = Slice ( * key_iter ) ;
value = Slice ( * value_iter ) ;
// We have a list of keys to write, traverse the list.
while ( true ) {
ParseInternalKey ( key , & ikey ) ;
status = WriteKeyValue ( key , value , ikey , input - > status ( ) ) ;
if ( ! status . ok ( ) ) {
break ;
}
+ + key_iter ;
+ + value_iter ;
// If at end of list
if ( key_iter = = keys . rend ( ) | | value_iter = = values . rend ( ) ) {
// Sanity Check: if one ends, then both end
assert ( key_iter = = keys . rend ( ) & & value_iter = = values . rend ( ) ) ;
break ;
}
// Otherwise not at end of list. Update key, value, and ikey.
key = Slice ( * key_iter ) ;
value = Slice ( * value_iter ) ;
ParseInternalKey ( key , & ikey ) ;
}
} else {
// There is only one item to be written out
status = WriteKeyValue ( key , value , ikey , input - > status ( ) ) ;
}
} // if (!drop)
// MergeUntil has moved input to the next entry
if ( ! current_entry_is_merging ) {
input - > Next ( ) ;
}
last_sequence_for_key = ikey . sequence ;
visible_in_snapshot = visible ;
}
RecordTick ( stats_ , FILTER_OPERATION_TOTAL_TIME , total_filter_time ) ;
RecordDroppedKeys ( & key_drop_user , & key_drop_newer_entry , & key_drop_obsolete ) ;
RecordCompactionIOStats ( ) ;
if ( status . ok ( ) & &
( shutting_down_ - > load ( std : : memory_order_acquire ) | | cfd - > IsDropped ( ) ) ) {
status = Status : : ShutdownInProgress (
" Database shutdown or Column family drop during compaction " ) ;
}
if ( status . ok ( ) & & compact_ - > builder ! = nullptr ) {
status = FinishCompactionOutputFile ( input - > status ( ) ) ;
}
if ( status . ok ( ) ) {
status = input - > status ( ) ;
}
return status ;
}
Boyang Zhang
10 years ago