@ -28,49 +28,33 @@ namespace ROCKSDB_NAMESPACE {
// TODO(icanadi) Mock out everything else:
// 1. VersionSet
// 2. Memtable
class FlushJobTest : public testing : : Test {
public :
FlushJobTest ( )
class FlushJobTestBase : public testing : : Test {
protected :
FlushJobTestBase ( std : : string dbname , const Comparator * ucmp )
: env_ ( Env : : Default ( ) ) ,
fs_ ( std : : make_shared < LegacyFileSystemWrapper > ( env_ ) ) ,
dbname_ ( test : : PerThreadDBPath ( " flush_job_test " ) ) ,
dbname_ ( std : : move ( dbname ) ) ,
ucmp_ ( ucmp ) ,
options_ ( ) ,
db_options_ ( options_ ) ,
column_family_names_ ( { kDefaultColumnFamilyName , " foo " , " bar " } ) ,
table_cache_ ( NewLRUCache ( 50000 , 16 ) ) ,
write_buffer_manager_ ( db_options_ . db_write_buffer_size ) ,
shutting_down_ ( false ) ,
mock_table_factory_ ( new mock : : MockTableFactory ( ) ) {
EXPECT_OK ( env_ - > CreateDirIfMissing ( dbname_ ) ) ;
db_options_ . db_paths . emplace_back ( dbname_ ,
std : : numeric_limits < uint64_t > : : max ( ) ) ;
db_options_ . statistics = ROCKSDB_NAMESPACE : : CreateDBStatistics ( ) ;
// TODO(icanadi) Remove this once we mock out VersionSet
NewDB ( ) ;
std : : vector < ColumnFamilyDescriptor > column_families ;
cf_options_ . table_factory = mock_table_factory_ ;
for ( const auto & cf_name : column_family_names_ ) {
column_families . emplace_back ( cf_name , cf_options_ ) ;
}
mock_table_factory_ ( new mock : : MockTableFactory ( ) ) { }
db_options_ . env = env_ ;
db_options_ . fs = fs_ ;
versions_ . reset (
new VersionSet ( dbname_ , & db_options_ , env_options_ , table_cache_ . get ( ) ,
& write_buffer_manager_ , & write_controller_ ,
/*block_cache_tracer=*/ nullptr , /*io_tracer=*/ nullptr ) ) ;
EXPECT_OK ( versions_ - > Recover ( column_families , false ) ) ;
virtual ~ FlushJobTestBase ( ) {
if ( getenv ( " KEEP_DB " ) ) {
fprintf ( stdout , " db is still in %s \n " , dbname_ . c_str ( ) ) ;
} else {
EXPECT_OK ( DestroyDir ( env_ , dbname_ ) ) ;
}
}
void NewDB ( ) {
SetIdentityFile ( env_ , dbname_ ) ;
VersionEdit new_db ;
if ( db_options_ . write_dbid_to_manifest ) {
DBImpl * impl = new DBImpl ( DBOptions ( ) , dbname_ ) ;
std : : string db_id ;
impl - > GetDbIdentityFromIdentityFile ( & db_id ) ;
new_db . SetDBId ( db_id ) ;
}
new_db . SetLogNumber ( 0 ) ;
new_db . SetNextFile ( 2 ) ;
new_db . SetLastSequence ( 0 ) ;
@ -82,6 +66,7 @@ class FlushJobTest : public testing::Test {
VersionEdit new_cf ;
new_cf . AddColumnFamily ( column_family_names_ [ i ] ) ;
new_cf . SetColumnFamily ( cf_id + + ) ;
new_cf . SetComparatorName ( ucmp_ - > Name ( ) ) ;
new_cf . SetLogNumber ( 0 ) ;
new_cf . SetNextFile ( 2 ) ;
new_cf . SetLastSequence ( last_seq + + ) ;
@ -114,9 +99,37 @@ class FlushJobTest : public testing::Test {
ASSERT_OK ( s ) ;
}
void SetUp ( ) override {
EXPECT_OK ( env_ - > CreateDirIfMissing ( dbname_ ) ) ;
// TODO(icanadi) Remove this once we mock out VersionSet
NewDB ( ) ;
db_options_ . env = env_ ;
db_options_ . fs = fs_ ;
db_options_ . db_paths . emplace_back ( dbname_ ,
std : : numeric_limits < uint64_t > : : max ( ) ) ;
db_options_ . statistics = CreateDBStatistics ( ) ;
cf_options_ . comparator = ucmp_ ;
std : : vector < ColumnFamilyDescriptor > column_families ;
cf_options_ . table_factory = mock_table_factory_ ;
for ( const auto & cf_name : column_family_names_ ) {
column_families . emplace_back ( cf_name , cf_options_ ) ;
}
versions_ . reset (
new VersionSet ( dbname_ , & db_options_ , env_options_ , table_cache_ . get ( ) ,
& write_buffer_manager_ , & write_controller_ ,
/*block_cache_tracer=*/ nullptr , /*io_tracer=*/ nullptr ) ) ;
EXPECT_OK ( versions_ - > Recover ( column_families , false ) ) ;
}
Env * env_ ;
std : : shared_ptr < FileSystem > fs_ ;
std : : string dbname_ ;
const Comparator * const ucmp_ ;
EnvOptions env_options_ ;
Options options_ ;
ImmutableDBOptions db_options_ ;
@ -131,6 +144,13 @@ class FlushJobTest : public testing::Test {
std : : shared_ptr < mock : : MockTableFactory > mock_table_factory_ ;
} ;
class FlushJobTest : public FlushJobTestBase {
public :
FlushJobTest ( )
: FlushJobTestBase ( test : : PerThreadDBPath ( " flush_job_test " ) ,
BytewiseComparator ( ) ) { }
} ;
TEST_F ( FlushJobTest , Empty ) {
JobContext job_context ( 0 ) ;
auto cfd = versions_ - > GetColumnFamilySet ( ) - > GetDefault ( ) ;
@ -487,6 +507,135 @@ TEST_F(FlushJobTest, Snapshots) {
job_context . Clean ( ) ;
}
class FlushJobTimestampTest : public FlushJobTestBase {
public :
FlushJobTimestampTest ( )
: FlushJobTestBase ( test : : PerThreadDBPath ( " flush_job_ts_gc_test " ) ,
test : : ComparatorWithU64Ts ( ) ) { }
void AddKeyValueToMemtable ( MemTable * memtable , std : : string key , uint64_t ts ,
SequenceNumber seq , ValueType value_type ,
Slice value ) {
std : : string key_str ( std : : move ( key ) ) ;
PutFixed64 ( & key_str , ts ) ;
memtable - > Add ( seq , value_type , key_str , value ) ;
}
protected :
static constexpr uint64_t kStartTs = 10 ;
static constexpr SequenceNumber kStartSeq = 0 ;
SequenceNumber curr_seq_ { kStartSeq } ;
std : : atomic < uint64_t > curr_ts_ { kStartTs } ;
} ;
TEST_F ( FlushJobTimestampTest , AllKeysExpired ) {
ColumnFamilyData * cfd = versions_ - > GetColumnFamilySet ( ) - > GetDefault ( ) ;
autovector < MemTable * > to_delete ;
{
MemTable * new_mem = cfd - > ConstructNewMemtable (
* cfd - > GetLatestMutableCFOptions ( ) , kMaxSequenceNumber ) ;
new_mem - > Ref ( ) ;
for ( int i = 0 ; i < 100 ; + + i ) {
uint64_t ts = curr_ts_ . fetch_add ( 1 ) ;
SequenceNumber seq = ( curr_seq_ + + ) ;
AddKeyValueToMemtable ( new_mem , test : : EncodeInt ( 0 ) , ts , seq ,
ValueType : : kTypeValue , " 0_value " ) ;
}
uint64_t ts = curr_ts_ . fetch_add ( 1 ) ;
SequenceNumber seq = ( curr_seq_ + + ) ;
AddKeyValueToMemtable ( new_mem , test : : EncodeInt ( 0 ) , ts , seq ,
ValueType : : kTypeDeletionWithTimestamp , " " ) ;
cfd - > imm ( ) - > Add ( new_mem , & to_delete ) ;
}
std : : vector < SequenceNumber > snapshots ;
constexpr SnapshotChecker * const snapshot_checker = nullptr ;
JobContext job_context ( 0 ) ;
EventLogger event_logger ( db_options_ . info_log . get ( ) ) ;
std : : string full_history_ts_low ;
PutFixed64 ( & full_history_ts_low , std : : numeric_limits < uint64_t > : : max ( ) ) ;
FlushJob flush_job (
dbname_ , cfd , db_options_ , * cfd - > GetLatestMutableCFOptions ( ) ,
nullptr /* memtable_id */ , env_options_ , versions_ . get ( ) , & mutex_ ,
& shutting_down_ , snapshots , kMaxSequenceNumber , snapshot_checker ,
& job_context , nullptr , nullptr , nullptr , kNoCompression ,
db_options_ . statistics . get ( ) , & event_logger , true ,
true /* sync_output_directory */ , true /* write_manifest */ ,
Env : : Priority : : USER , nullptr /*IOTracer*/ , /*db_id=*/ " " ,
/*db_session_id=*/ " " , full_history_ts_low ) ;
FileMetaData fmeta ;
mutex_ . Lock ( ) ;
flush_job . PickMemTable ( ) ;
ASSERT_OK ( flush_job . Run ( /*prep_tracker=*/ nullptr , & fmeta ) ) ;
mutex_ . Unlock ( ) ;
{
std : : string key = test : : EncodeInt ( 0 ) ;
key . append ( test : : EncodeInt ( curr_ts_ . load ( std : : memory_order_relaxed ) - 1 ) ) ;
InternalKey ikey ( key , curr_seq_ - 1 , ValueType : : kTypeDeletionWithTimestamp ) ;
ASSERT_EQ ( ikey . Encode ( ) , fmeta . smallest . Encode ( ) ) ;
ASSERT_EQ ( ikey . Encode ( ) , fmeta . largest . Encode ( ) ) ;
}
job_context . Clean ( ) ;
ASSERT_TRUE ( to_delete . empty ( ) ) ;
}
TEST_F ( FlushJobTimestampTest , NoKeyExpired ) {
ColumnFamilyData * cfd = versions_ - > GetColumnFamilySet ( ) - > GetDefault ( ) ;
autovector < MemTable * > to_delete ;
{
MemTable * new_mem = cfd - > ConstructNewMemtable (
* cfd - > GetLatestMutableCFOptions ( ) , kMaxSequenceNumber ) ;
new_mem - > Ref ( ) ;
for ( int i = 0 ; i < 100 ; + + i ) {
uint64_t ts = curr_ts_ . fetch_add ( 1 ) ;
SequenceNumber seq = ( curr_seq_ + + ) ;
AddKeyValueToMemtable ( new_mem , test : : EncodeInt ( 0 ) , ts , seq ,
ValueType : : kTypeValue , " 0_value " ) ;
}
cfd - > imm ( ) - > Add ( new_mem , & to_delete ) ;
}
std : : vector < SequenceNumber > snapshots ;
SnapshotChecker * const snapshot_checker = nullptr ;
JobContext job_context ( 0 ) ;
EventLogger event_logger ( db_options_ . info_log . get ( ) ) ;
std : : string full_history_ts_low ;
PutFixed64 ( & full_history_ts_low , 0 ) ;
FlushJob flush_job (
dbname_ , cfd , db_options_ , * cfd - > GetLatestMutableCFOptions ( ) ,
nullptr /* memtable_id */ , env_options_ , versions_ . get ( ) , & mutex_ ,
& shutting_down_ , snapshots , kMaxSequenceNumber , snapshot_checker ,
& job_context , nullptr , nullptr , nullptr , kNoCompression ,
db_options_ . statistics . get ( ) , & event_logger , true ,
true /* sync_output_directory */ , true /* write_manifest */ ,
Env : : Priority : : USER , nullptr /*IOTracer*/ , /*db_id=*/ " " ,
/*db_session_id=*/ " " , full_history_ts_low ) ;
FileMetaData fmeta ;
mutex_ . Lock ( ) ;
flush_job . PickMemTable ( ) ;
ASSERT_OK ( flush_job . Run ( /*prep_tracker=*/ nullptr , & fmeta ) ) ;
mutex_ . Unlock ( ) ;
{
std : : string ukey = test : : EncodeInt ( 0 ) ;
std : : string smallest_key =
ukey + test : : EncodeInt ( curr_ts_ . load ( std : : memory_order_relaxed ) - 1 ) ;
std : : string largest_key = ukey + test : : EncodeInt ( kStartTs ) ;
InternalKey smallest ( smallest_key , curr_seq_ - 1 , ValueType : : kTypeValue ) ;
InternalKey largest ( largest_key , kStartSeq , ValueType : : kTypeValue ) ;
ASSERT_EQ ( smallest . Encode ( ) , fmeta . smallest . Encode ( ) ) ;
ASSERT_EQ ( largest . Encode ( ) , fmeta . largest . Encode ( ) ) ;
}
job_context . Clean ( ) ;
ASSERT_TRUE ( to_delete . empty ( ) ) ;
}
} // namespace ROCKSDB_NAMESPACE
int main ( int argc , char * * argv ) {
Yanqin Jin
4 years ago
committed by