@ -9,6 +9,7 @@
# include "db/builder.h" # include "db/builder.h"
# include <algorithm>
# include <deque> # include <deque>
# include <vector> # include <vector>
@ -31,6 +32,28 @@
namespace rocksdb { namespace rocksdb {
namespace {
inline SequenceNumber EarliestVisibleSnapshot (
SequenceNumber in , const std : : vector < SequenceNumber > & snapshots ,
SequenceNumber * prev_snapshot ) {
if ( snapshots . empty ( ) ) {
* prev_snapshot = 0 ; // 0 means no previous snapshot
return kMaxSequenceNumber ;
}
SequenceNumber prev = 0 ;
for ( const auto cur : snapshots ) {
assert ( prev < = cur ) ;
if ( cur > = in ) {
* prev_snapshot = prev ;
return cur ;
}
prev = cur ; // assignment
}
* prev_snapshot = prev ;
return kMaxSequenceNumber ;
}
} // namespace
class TableFactory ; class TableFactory ;
TableBuilder * NewTableBuilder ( TableBuilder * NewTableBuilder (
@ -53,9 +76,7 @@ Status BuildTable(
FileMetaData * meta , const InternalKeyComparator & internal_comparator , FileMetaData * meta , const InternalKeyComparator & internal_comparator ,
const std : : vector < std : : unique_ptr < IntTblPropCollectorFactory > > * const std : : vector < std : : unique_ptr < IntTblPropCollectorFactory > > *
int_tbl_prop_collector_factories , int_tbl_prop_collector_factories ,
const SequenceNumber newest_snapshot , std : : vector < SequenceNumber > snapshots , const CompressionType compression ,
const SequenceNumber earliest_seqno_in_memtable ,
const CompressionType compression ,
const CompressionOptions & compression_opts , bool paranoid_file_checks , const CompressionOptions & compression_opts , bool paranoid_file_checks ,
InternalStats * internal_stats , const Env : : IOPriority io_priority , InternalStats * internal_stats , const Env : : IOPriority io_priority ,
TableProperties * table_properties ) { TableProperties * table_properties ) {
@ -66,14 +87,6 @@ Status BuildTable(
meta - > smallest_seqno = meta - > largest_seqno = 0 ; meta - > smallest_seqno = meta - > largest_seqno = 0 ;
iter - > SeekToFirst ( ) ; iter - > SeekToFirst ( ) ;
// If the sequence number of the smallest entry in the memtable is
// smaller than the most recent snapshot, then we do not trigger
// removal of duplicate/deleted keys as part of this builder.
bool purge = true ;
if ( earliest_seqno_in_memtable < = newest_snapshot ) {
purge = false ;
}
std : : string fname = TableFileName ( ioptions . db_paths , meta - > fd . GetNumber ( ) , std : : string fname = TableFileName ( ioptions . db_paths , meta - > fd . GetNumber ( ) ,
meta - > fd . GetPathId ( ) ) ; meta - > fd . GetPathId ( ) ) ;
if ( iter - > Valid ( ) ) { if ( iter - > Valid ( ) ) {
@ -107,40 +120,62 @@ Status BuildTable(
ioptions . min_partial_merge_operands , ioptions . min_partial_merge_operands ,
true /* internal key corruption is not ok */ ) ; true /* internal key corruption is not ok */ ) ;
if ( purge ) { IterKey current_user_key ;
// Ugly walkaround to avoid compiler error for release build
bool has_current_user_key = false ;
bool ok __attribute__ ( ( unused ) ) = true ; // If has_current_user_key == true, this variable remembers the earliest
// snapshot in which this current key already exists. If two internal keys
// Will write to builder if current key != prev key
// have the same user key AND the earlier one should be visible in the
ParsedInternalKey prev_ikey ; // snapshot in which we already have a user key, we can drop the earlier
std : : string prev_key ; // user key
bool is_first_key = true ; // Also write if this is the very first key
SequenceNumber current_user_key_exists_in_snapshot = kMaxSequenceNumber ;
while ( iter - > Valid ( ) ) { while ( iter - > Valid ( ) ) {
bool iterator_at_next = false ;
// Get current key
// Get current key
ParsedInternalKey this_ ikey; ParsedInternalKey ikey ;
Slice key = iter - > key ( ) ; Slice key = iter - > key ( ) ;
Slice value = iter - > value ( ) ; Slice value = iter - > value ( ) ;
// In-memory key corruption is not ok;
// In-memory key corruption is not ok;
// TODO: find a clean way to treat in memory key corruption
// TODO: find a clean way to treat in memory key corruption
ok = ParseInternalKey ( key , & this_ikey ) ; // Ugly walkaround to avoid compiler error for release build
bool ok __attribute__ ( ( unused ) ) = true ;
ok = ParseInternalKey ( key , & ikey ) ;
assert ( ok ) ; assert ( ok ) ;
assert ( this_ikey . sequence > = earliest_seqno_in_memtable ) ;
meta - > smallest_seqno = std : : min ( meta - > smallest_seqno , ikey . sequence ) ;
meta - > largest_seqno = std : : max ( meta - > largest_seqno , ikey . sequence ) ;
// If the key is the same as the previous key (and it is not the
// If the key is the same as the previous key (and it is not the
// first key), then we skip it, since it is an older version.
// first key), then we skip it, since it is an older version.
// Otherwise we output the key and mark it as the "new" previous key.
// Otherwise we output the key and mark it as the "new" previous key.
if ( ! is_first_key & & ! internal_comparator . user_comparator ( ) - > Compare ( if ( ! has_current_user_key | |
prev_ikey . user_key , this_ikey . user_key ) ) { internal_comparator . user_comparator ( ) - > Compare (
// seqno within the same key are in decreasing order
ikey . user_key , current_user_key . GetKey ( ) ) ! = 0 ) {
assert ( this_ikey . sequence < prev_ikey . sequence ) ; // First occurrence of this user key
} else { current_user_key . SetKey ( ikey . user_key ) ;
is_first_key = false ; has_current_user_key = true ;
current_user_key_exists_in_snapshot = 0 ;
}
// If there are no snapshots, then this kv affect visibility at tip.
// Otherwise, search though all existing snapshots to find
// the earlist snapshot that is affected by this kv.
SequenceNumber prev_snapshot = 0 ; // 0 means no previous snapshot
SequenceNumber key_needs_to_exist_in_snapshot =
EarliestVisibleSnapshot ( ikey . sequence , snapshots , & prev_snapshot ) ;
if ( current_user_key_exists_in_snapshot = =
key_needs_to_exist_in_snapshot ) {
// If this user key already exists in snapshot in which it needs to
// exist, we can drop it.
// In other words, if the earliest snapshot is which this key is visible
// in is the same as the visibily of a previous instance of the
// same key, then this kv is not visible in any snapshot.
// Hidden by an newer entry for same user key
iter - > Next ( ) ;
} else if ( ikey . type = = kTypeMerge ) {
meta - > largest . DecodeFrom ( key ) ;
if ( this_ikey . type = = kTypeMerge ) {
// TODO(tbd): Add a check here to prevent RocksDB from crash when
// TODO(tbd): Add a check here to prevent RocksDB from crash when
// reopening a DB w/o properly specifying the merge operator. But
// reopening a DB w/o properly specifying the merge operator. But
// currently we observed a memory leak on failing in RocksDB
// currently we observed a memory leak on failing in RocksDB
@ -150,71 +185,49 @@ Status BuildTable(
// Handle merge-type keys using the MergeHelper
// Handle merge-type keys using the MergeHelper
// TODO: pass statistics to MergeUntil
// TODO: pass statistics to MergeUntil
merge . MergeUntil ( iter , 0 /* don't worry about snapshot */ ) ; merge . MergeUntil ( iter , prev_snapshot , false , nullptr , env ) ;
iterator_at_next = true ; // IMPORTANT: Slice key doesn't point to a valid value anymore!!
// Write them out one-by-one. (Proceed back() to front())
const auto & keys = merge . keys ( ) ;
// If the merge successfully merged the input into
const auto & values = merge . values ( ) ;
// a kTypeValue, the list contains a single element.
assert ( ! keys . empty ( ) ) ;
const std : : deque < std : : string > & keys = merge . keys ( ) ; assert ( keys . size ( ) = = values . size ( ) ) ;
const std : : deque < std : : string > & values = merge . values ( ) ;
assert ( keys . size ( ) = = values . size ( ) & & keys . size ( ) > = 1 ) ; // largest possible sequence number in a merge queue is already stored
std : : deque < std : : string > : : const_reverse_iterator key_iter ; // in ikey.sequence.
std : : deque < std : : string > : : const_reverse_iterator value_iter ; // we additionally have to consider the front of the merge queue, which
for ( key_iter = keys . rbegin ( ) , value_iter = values . rbegin ( ) ; // might have the smallest sequence number (out of all the merges with
key_iter ! = keys . rend ( ) & & value_iter ! = values . rend ( ) ; // the same key)
+ + key_iter , + + value_iter ) { meta - > smallest_seqno =
builder - > Add ( Slice ( * key_iter ) , Slice ( * value_iter ) ) ; std : : min ( meta - > smallest_seqno , GetInternalKeySeqno ( keys . front ( ) ) ) ;
}
// We have a list of keys to write, write all keys in the list.
// Sanity check. Both iterators should end at the same time
for ( auto key_iter = keys . rbegin ( ) , value_iter = values . rbegin ( ) ;
assert ( key_iter = = keys . rend ( ) & & value_iter = = values . rend ( ) ) ; key_iter ! = keys . rend ( ) ; key_iter + + , value_iter + + ) {
key = Slice ( * key_iter ) ;
prev_key . assign ( keys . front ( ) ) ; value = Slice ( * value_iter ) ;
ok = ParseInternalKey ( Slice ( prev_key ) , & prev_ikey ) ; bool valid_key __attribute__ ( ( __unused__ ) ) =
assert ( ok ) ; ParseInternalKey ( key , & ikey ) ;
} else { // MergeUntil stops when it encounters a corrupt key and does not
// Handle Put/Delete-type keys by simply writing them
// include them in the result, so we expect the keys here to valid.
assert ( valid_key ) ;
builder - > Add ( key , value ) ; builder - > Add ( key , value ) ;
prev_key . assign ( key . data ( ) , key . size ( ) ) ;
ok = ParseInternalKey ( Slice ( prev_key ) , & prev_ikey ) ;
assert ( ok ) ;
}
}
if ( io_priority = = Env : : IO_HIGH & &
IOSTATS ( bytes_written ) > = kReportFlushIOStatsEvery ) {
ThreadStatusUtil : : IncreaseThreadOperationProperty (
ThreadStatus : : FLUSH_BYTES_WRITTEN ,
IOSTATS ( bytes_written ) ) ;
IOSTATS_RESET ( bytes_written ) ;
} }
if ( ! iterator_at_next ) iter - > Next ( ) ; } else { // just write out the key-value
builder - > Add ( key , value ) ;
meta - > largest . DecodeFrom ( key ) ;
iter - > Next ( ) ;
} }
// The last key is the largest key
current_user_key_exists_in_snapshot = key_needs_to_exist_in_snapshot ;
meta - > largest . DecodeFrom ( Slice ( prev_key ) ) ;
SequenceNumber seqno = GetInternalKeySeqno ( Slice ( prev_key ) ) ;
meta - > smallest_seqno = std : : min ( meta - > smallest_seqno , seqno ) ;
meta - > largest_seqno = std : : max ( meta - > largest_seqno , seqno ) ;
} else {
for ( ; iter - > Valid ( ) ; iter - > Next ( ) ) {
Slice key = iter - > key ( ) ;
meta - > largest . DecodeFrom ( key ) ;
builder - > Add ( key , iter - > value ( ) ) ;
SequenceNumber seqno = GetInternalKeySeqno ( key ) ;
meta - > smallest_seqno = std : : min ( meta - > smallest_seqno , seqno ) ;
meta - > largest_seqno = std : : max ( meta - > largest_seqno , seqno ) ;
if ( io_priority = = Env : : IO_HIGH & & if ( io_priority = = Env : : IO_HIGH & &
IOSTATS ( bytes_written ) > = kReportFlushIOStatsEvery ) { IOSTATS ( bytes_written ) > = kReportFlushIOStatsEvery ) {
ThreadStatusUtil : : IncreaseThreadOperationProperty ( ThreadStatusUtil : : IncreaseThreadOperationProperty (
ThreadStatus : : FLUSH_BYTES_WRITTEN , ThreadStatus : : FLUSH_BYTES_WRITTEN , IOSTATS ( bytes_written ) ) ;
IOSTATS ( bytes_written ) ) ;
IOSTATS_RESET ( bytes_written ) ; IOSTATS_RESET ( bytes_written ) ;
} }
} }
}
// Finish and check for builder errors
// Finish and check for builder errors
if ( s . ok ( ) ) { if ( s . ok ( ) ) {
Igor Canadi
9 years ago