@ -250,9 +250,7 @@ class CompactionJobTest : public testing::Test {
const stl_wrappers : : KVMap & expected_results ,
const std : : vector < SequenceNumber > & snapshots = { } ,
SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber ,
int output_level = 1 , bool verify = true ,
SnapshotListFetchCallback * snapshot_fetcher =
SnapshotListFetchCallback : : kDisabled ) {
int output_level = 1 , bool verify = true ) {
auto cfd = versions_ - > GetColumnFamilySet ( ) - > GetDefault ( ) ;
size_t num_input_files = 0 ;
@ -285,7 +283,7 @@ class CompactionJobTest : public testing::Test {
nullptr , nullptr , & mutex_ , & error_handler_ , snapshots ,
earliest_write_conflict_snapshot , snapshot_checker , table_cache_ ,
& event_logger , false , false , dbname_ , & compaction_job_stats_ ,
Env : : Priority : : USER , snapshot_fetcher ) ;
Env : : Priority : : USER ) ;
VerifyInitializationOfCompactionJobStats ( compaction_job_stats_ ) ;
compaction_job . Prepare ( ) ;
@ -962,105 +960,6 @@ TEST_F(CompactionJobTest, CorruptionAfterDeletion) {
RunCompaction ( { files } , expected_results ) ;
}
// Test the snapshot fetcher in compaction
TEST_F ( CompactionJobTest , SnapshotRefresh ) {
uint64_t time_seed = env_ - > NowMicros ( ) ;
printf ( " time_seed is % " PRIu64 " \n " , time_seed ) ; // would help to reproduce
Random64 rand ( time_seed ) ;
std : : vector < SequenceNumber > db_snapshots ;
class SnapshotListFetchCallbackTest : public SnapshotListFetchCallback {
public :
SnapshotListFetchCallbackTest ( Env * env , Random64 & rand ,
std : : vector < SequenceNumber > * snapshots )
: SnapshotListFetchCallback ( env , 1 /*short time delay*/ ,
1 /*fetch after each key*/ ) ,
rand_ ( rand ) ,
snapshots_ ( snapshots ) { }
virtual void Refresh ( std : : vector < SequenceNumber > * snapshots ,
SequenceNumber ) override {
assert ( snapshots - > size ( ) ) ;
assert ( snapshots_ - > size ( ) ) ;
assert ( snapshots_ - > size ( ) = = snapshots - > size ( ) ) ;
if ( rand_ . OneIn ( 2 ) ) {
uint64_t release_index = rand_ . Uniform ( snapshots_ - > size ( ) ) ;
snapshots_ - > erase ( snapshots_ - > begin ( ) + release_index ) ;
* snapshots = * snapshots_ ;
}
}
private :
Random64 rand_ ;
std : : vector < SequenceNumber > * snapshots_ ;
} snapshot_fetcher ( env_ , rand , & db_snapshots ) ;
std : : vector < std : : pair < const std : : string , std : : string > > file1_kvs , file2_kvs ;
std : : array < ValueType , 4 > types = { kTypeValue , kTypeDeletion ,
kTypeSingleDeletion } ;
SequenceNumber last_seq = 0 ;
for ( int i = 1 ; i < 100 ; i + + ) {
SequenceNumber seq = last_seq + 1 ;
last_seq = seq ;
if ( rand . OneIn ( 2 ) ) {
auto type = types [ rand . Uniform ( types . size ( ) ) ] ;
file1_kvs . push_back (
{ test : : KeyStr ( " k " + ToString ( i ) , seq , type ) , " v " + ToString ( i ) } ) ;
}
}
auto file1 = mock : : MakeMockFile ( file1_kvs ) ;
for ( int i = 1 ; i < 100 ; i + + ) {
SequenceNumber seq = last_seq + 1 ;
last_seq + + ;
if ( rand . OneIn ( 2 ) ) {
auto type = types [ rand . Uniform ( types . size ( ) ) ] ;
file2_kvs . push_back (
{ test : : KeyStr ( " k " + ToString ( i ) , seq , type ) , " v " + ToString ( i ) } ) ;
}
}
auto file2 = mock : : MakeMockFile ( file2_kvs ) ;
for ( SequenceNumber i = 1 ; i < last_seq + 1 ; i + + ) {
if ( rand . OneIn ( 5 ) ) {
db_snapshots . push_back ( i ) ;
}
}
const bool kVerify = true ;
const int output_level_0 = 0 ;
NewDB ( ) ;
AddMockFile ( file1 ) ;
AddMockFile ( file2 ) ;
SetLastSequence ( last_seq ) ;
auto files = cfd_ - > current ( ) - > storage_info ( ) - > LevelFiles ( 0 ) ;
// put the output on L0 since it is easier to feed them again to the 2nd
// compaction
RunCompaction ( { files } , file1 , db_snapshots , kMaxSequenceNumber ,
output_level_0 , ! kVerify , & snapshot_fetcher ) ;
// Now db_snapshots are changed. Run the compaction again without snapshot
// fetcher but with the updated snapshot list.
compaction_job_stats_ . Reset ( ) ;
files = cfd_ - > current ( ) - > storage_info ( ) - > LevelFiles ( 0 ) ;
RunCompaction ( { files } , file1 , db_snapshots , kMaxSequenceNumber ,
output_level_0 + 1 , ! kVerify ) ;
// The result should be what we get if we run compaction without snapshot
// fetcher on the updated list of snapshots
auto expected = mock_table_factory_ - > output ( ) ;
NewDB ( ) ;
AddMockFile ( file1 ) ;
AddMockFile ( file2 ) ;
SetLastSequence ( last_seq ) ;
files = cfd_ - > current ( ) - > storage_info ( ) - > LevelFiles ( 0 ) ;
RunCompaction ( { files } , expected , db_snapshots , kMaxSequenceNumber ,
output_level_0 , ! kVerify ) ;
// The 2nd compaction above would get rid of useless delete markers. To get
// the output here exactly as what we got above after two compactions, we also
// run the compaction for 2nd time.
compaction_job_stats_ . Reset ( ) ;
files = cfd_ - > current ( ) - > storage_info ( ) - > LevelFiles ( 0 ) ;
RunCompaction ( { files } , expected , db_snapshots , kMaxSequenceNumber ,
output_level_0 + 1 , ! kVerify ) ;
}
} // namespace rocksdb
int main ( int argc , char * * argv ) {
Maysam Yabandeh
5 years ago
committed by