@ -777,13 +777,25 @@ TEST_F(DBFlushTest, MemPurgeBasic) {
// Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes).
options . write_buffer_size = 1 < < 20 ;
// Activate the MemPurge prototype.
options . experimental_mempurge_threshold = 1.0 ;
# ifndef ROCKSDB_LITE
// Initially deactivate the MemPurge prototype.
options . experimental_mempurge_threshold = 0.0 ;
TestFlushListener * listener = new TestFlushListener ( options . env , this ) ;
options . listeners . emplace_back ( listener ) ;
# else
// Activate directly the MemPurge prototype.
// (RocksDB lite does not support dynamic options)
options . experimental_mempurge_threshold = 1.0 ;
# endif // !ROCKSDB_LITE
ASSERT_OK ( TryReopen ( options ) ) ;
// RocksDB lite does not support dynamic options
# ifndef ROCKSDB_LITE
// Dynamically activate the MemPurge prototype without restarting the DB.
ColumnFamilyHandle * cfh = db_ - > DefaultColumnFamily ( ) ;
ASSERT_OK ( db_ - > SetOptions ( cfh , { { " experimental_mempurge_threshold " , " 1.0 " } } ) ) ;
# endif
std : : atomic < uint32_t > mempurge_count { 0 } ;
std : : atomic < uint32_t > sst_count { 0 } ;
ROCKSDB_NAMESPACE : : SyncPoint : : GetInstance ( ) - > SetCallBack (
@ -914,6 +926,234 @@ TEST_F(DBFlushTest, MemPurgeBasic) {
Close ( ) ;
}
// RocksDB lite does not support dynamic options
# ifndef ROCKSDB_LITE
TEST_F ( DBFlushTest , MemPurgeBasicToggle ) {
Options options = CurrentOptions ( ) ;
// The following options are used to enforce several values that
// may already exist as default values to make this test resilient
// to default value updates in the future.
options . statistics = CreateDBStatistics ( ) ;
// Record all statistics.
options . statistics - > set_stats_level ( StatsLevel : : kAll ) ;
// create the DB if it's not already present
options . create_if_missing = true ;
// Useful for now as we are trying to compare uncompressed data savings on
// flush().
options . compression = kNoCompression ;
// Prevent memtable in place updates. Should already be disabled
// (from Wiki:
// In place updates can be enabled by toggling on the bool
// inplace_update_support flag. However, this flag is by default set to
// false
// because this thread-safe in-place update support is not compatible
// with concurrent memtable writes. Note that the bool
// allow_concurrent_memtable_write is set to true by default )
options . inplace_update_support = false ;
options . allow_concurrent_memtable_write = true ;
// Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes).
options . write_buffer_size = 1 < < 20 ;
// Initially deactivate the MemPurge prototype.
// (negative values are equivalent to 0.0).
options . experimental_mempurge_threshold = - 25.3 ;
TestFlushListener * listener = new TestFlushListener ( options . env , this ) ;
options . listeners . emplace_back ( listener ) ;
ASSERT_OK ( TryReopen ( options ) ) ;
// Dynamically activate the MemPurge prototype without restarting the DB.
ColumnFamilyHandle * cfh = db_ - > DefaultColumnFamily ( ) ;
// Values greater than 1.0 are equivalent to 1.0
ASSERT_OK (
db_ - > SetOptions ( cfh , { { " experimental_mempurge_threshold " , " 3.7898 " } } ) ) ;
std : : atomic < uint32_t > mempurge_count { 0 } ;
std : : atomic < uint32_t > sst_count { 0 } ;
ROCKSDB_NAMESPACE : : SyncPoint : : GetInstance ( ) - > SetCallBack (
" DBImpl::FlushJob:MemPurgeSuccessful " ,
[ & ] ( void * /*arg*/ ) { mempurge_count + + ; } ) ;
ROCKSDB_NAMESPACE : : SyncPoint : : GetInstance ( ) - > SetCallBack (
" DBImpl::FlushJob:SSTFileCreated " , [ & ] ( void * /*arg*/ ) { sst_count + + ; } ) ;
ROCKSDB_NAMESPACE : : SyncPoint : : GetInstance ( ) - > EnableProcessing ( ) ;
const size_t KVSIZE = 3 ;
std : : vector < std : : string > KEYS ( KVSIZE ) ;
for ( size_t k = 0 ; k < KVSIZE ; k + + ) {
KEYS [ k ] = " IamKey " + std : : to_string ( k ) ;
}
std : : vector < std : : string > RNDVALS ( KVSIZE ) ;
const std : : string NOT_FOUND = " NOT_FOUND " ;
// Heavy overwrite workload,
// more than would fit in maximum allowed memtables.
Random rnd ( 719 ) ;
const size_t NUM_REPEAT = 100 ;
const size_t RAND_VALUES_LENGTH = 10240 ;
// Insertion of of K-V pairs, multiple times (overwrites).
for ( size_t i = 0 ; i < NUM_REPEAT ; i + + ) {
for ( size_t j = 0 ; j < KEYS . size ( ) ; j + + ) {
RNDVALS [ j ] = rnd . RandomString ( RAND_VALUES_LENGTH ) ;
ASSERT_OK ( Put ( KEYS [ j ] , RNDVALS [ j ] ) ) ;
ASSERT_EQ ( Get ( KEYS [ j ] ) , RNDVALS [ j ] ) ;
}
for ( size_t j = 0 ; j < KEYS . size ( ) ; j + + ) {
ASSERT_EQ ( Get ( KEYS [ j ] ) , RNDVALS [ j ] ) ;
}
}
// Check that there was at least one mempurge
const uint32_t EXPECTED_MIN_MEMPURGE_COUNT = 1 ;
// Check that there was no SST files created during flush.
const uint32_t EXPECTED_SST_COUNT = 0 ;
EXPECT_GE ( mempurge_count . exchange ( 0 ) , EXPECTED_MIN_MEMPURGE_COUNT ) ;
EXPECT_EQ ( sst_count . exchange ( 0 ) , EXPECTED_SST_COUNT ) ;
// Dynamically deactivate MemPurge.
ASSERT_OK (
db_ - > SetOptions ( cfh , { { " experimental_mempurge_threshold " , " -1023.0 " } } ) ) ;
// Insertion of of K-V pairs, multiple times (overwrites).
for ( size_t i = 0 ; i < NUM_REPEAT ; i + + ) {
for ( size_t j = 0 ; j < KEYS . size ( ) ; j + + ) {
RNDVALS [ j ] = rnd . RandomString ( RAND_VALUES_LENGTH ) ;
ASSERT_OK ( Put ( KEYS [ j ] , RNDVALS [ j ] ) ) ;
ASSERT_EQ ( Get ( KEYS [ j ] ) , RNDVALS [ j ] ) ;
}
for ( size_t j = 0 ; j < KEYS . size ( ) ; j + + ) {
ASSERT_EQ ( Get ( KEYS [ j ] ) , RNDVALS [ j ] ) ;
}
}
// Check that there was at least one mempurge
const uint32_t ZERO = 0 ;
// Assert that at least one flush to storage has been performed
EXPECT_GT ( sst_count . exchange ( 0 ) , EXPECTED_SST_COUNT ) ;
// The mempurge count is expected to be set to 0 when the options are updated.
// We expect no mempurge at all.
EXPECT_EQ ( mempurge_count . exchange ( 0 ) , ZERO ) ;
Close ( ) ;
}
// Closes the "#ifndef ROCKSDB_LITE"
// End of MemPurgeBasicToggle, which is not
// supported with RocksDB LITE because it
// relies on dynamically changing the option
// flag experimental_mempurge_threshold.
# endif
// At the moment, MemPurge feature is deactivated
// when atomic_flush is enabled. This is because the level
// of garbage between Column Families is not guaranteed to
// be consistent, therefore a CF could hypothetically
// trigger a MemPurge while another CF would trigger
// a regular Flush.
TEST_F ( DBFlushTest , MemPurgeWithAtomicFlush ) {
Options options = CurrentOptions ( ) ;
// The following options are used to enforce several values that
// may already exist as default values to make this test resilient
// to default value updates in the future.
options . statistics = CreateDBStatistics ( ) ;
// Record all statistics.
options . statistics - > set_stats_level ( StatsLevel : : kAll ) ;
// create the DB if it's not already present
options . create_if_missing = true ;
// Useful for now as we are trying to compare uncompressed data savings on
// flush().
options . compression = kNoCompression ;
// Prevent memtable in place updates. Should already be disabled
// (from Wiki:
// In place updates can be enabled by toggling on the bool
// inplace_update_support flag. However, this flag is by default set to
// false
// because this thread-safe in-place update support is not compatible
// with concurrent memtable writes. Note that the bool
// allow_concurrent_memtable_write is set to true by default )
options . inplace_update_support = false ;
options . allow_concurrent_memtable_write = true ;
// Enforce size of a single MemTable to 64KB (64KB = 65,536 bytes).
options . write_buffer_size = 1 < < 20 ;
// Activate the MemPurge prototype.
options . experimental_mempurge_threshold = 153.245 ;
// Activate atomic_flush.
options . atomic_flush = true ;
const std : : vector < std : : string > new_cf_names = { " pikachu " , " eevie " } ;
CreateColumnFamilies ( new_cf_names , options ) ;
Close ( ) ;
// 3 CFs: default will be filled with overwrites (would normally trigger
// mempurge)
// new_cf_names[1] will be filled with random values (would trigger
// flush) new_cf_names[2] not filled with anything.
ReopenWithColumnFamilies (
{ kDefaultColumnFamilyName , new_cf_names [ 0 ] , new_cf_names [ 1 ] } , options ) ;
size_t num_cfs = handles_ . size ( ) ;
ASSERT_EQ ( 3 , num_cfs ) ;
ASSERT_OK ( Put ( 1 , " foo " , " bar " ) ) ;
ASSERT_OK ( Put ( 2 , " bar " , " baz " ) ) ;
std : : atomic < uint32_t > mempurge_count { 0 } ;
std : : atomic < uint32_t > sst_count { 0 } ;
ROCKSDB_NAMESPACE : : SyncPoint : : GetInstance ( ) - > SetCallBack (
" DBImpl::FlushJob:MemPurgeSuccessful " ,
[ & ] ( void * /*arg*/ ) { mempurge_count + + ; } ) ;
ROCKSDB_NAMESPACE : : SyncPoint : : GetInstance ( ) - > SetCallBack (
" DBImpl::FlushJob:SSTFileCreated " , [ & ] ( void * /*arg*/ ) { sst_count + + ; } ) ;
ROCKSDB_NAMESPACE : : SyncPoint : : GetInstance ( ) - > EnableProcessing ( ) ;
const size_t KVSIZE = 3 ;
std : : vector < std : : string > KEYS ( KVSIZE ) ;
for ( size_t k = 0 ; k < KVSIZE ; k + + ) {
KEYS [ k ] = " IamKey " + std : : to_string ( k ) ;
}
std : : string RNDKEY ;
std : : vector < std : : string > RNDVALS ( KVSIZE ) ;
const std : : string NOT_FOUND = " NOT_FOUND " ;
// Heavy overwrite workload,
// more than would fit in maximum allowed memtables.
Random rnd ( 106 ) ;
const size_t NUM_REPEAT = 100 ;
const size_t RAND_KEY_LENGTH = 128 ;
const size_t RAND_VALUES_LENGTH = 10240 ;
// Insertion of of K-V pairs, multiple times (overwrites).
for ( size_t i = 0 ; i < NUM_REPEAT ; i + + ) {
for ( size_t j = 0 ; j < KEYS . size ( ) ; j + + ) {
RNDKEY = rnd . RandomString ( RAND_KEY_LENGTH ) ;
RNDVALS [ j ] = rnd . RandomString ( RAND_VALUES_LENGTH ) ;
ASSERT_OK ( Put ( KEYS [ j ] , RNDVALS [ j ] ) ) ;
ASSERT_OK ( Put ( 1 , RNDKEY , RNDVALS [ j ] ) ) ;
ASSERT_EQ ( Get ( KEYS [ j ] ) , RNDVALS [ j ] ) ;
ASSERT_EQ ( Get ( 1 , RNDKEY ) , RNDVALS [ j ] ) ;
}
}
// Check that there was no mempurge because atomic_flush option is true.
const uint32_t EXPECTED_MIN_MEMPURGE_COUNT = 0 ;
// Check that there was at least one SST files created during flush.
const uint32_t EXPECTED_SST_COUNT = 1 ;
EXPECT_EQ ( mempurge_count . exchange ( 0 ) , EXPECTED_MIN_MEMPURGE_COUNT ) ;
EXPECT_GE ( sst_count . exchange ( 0 ) , EXPECTED_SST_COUNT ) ;
Close ( ) ;
}
TEST_F ( DBFlushTest , MemPurgeDeleteAndDeleteRange ) {
Options options = CurrentOptions ( ) ;
@ -930,7 +1170,7 @@ TEST_F(DBFlushTest, MemPurgeDeleteAndDeleteRange) {
// Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes).
options . write_buffer_size = 1 < < 20 ;
// Activate the MemPurge prototype.
options . experimental_mempurge_threshold = 1.0 ;
options . experimental_mempurge_threshold = 15 .0 ;
ASSERT_OK ( TryReopen ( options ) ) ;
@ -1137,7 +1377,7 @@ TEST_F(DBFlushTest, MemPurgeAndCompactionFilter) {
// Enforce size of a single MemTable to 64MB (64MB = 67108864 bytes).
options . write_buffer_size = 1 < < 20 ;
// Activate the MemPurge prototype.
options . experimental_mempurge_threshold = 1.0 ;
options . experimental_mempurge_threshold = 26.55 ;
ASSERT_OK ( TryReopen ( options ) ) ;
@ -1212,8 +1452,9 @@ TEST_F(DBFlushTest, DISABLED_MemPurgeWALSupport) {
// Enforce size of a single MemTable to 128KB.
options . write_buffer_size = 128 < < 10 ;
// Activate the MemPurge prototype.
options . experimental_mempurge_threshold = 1.0 ;
// Activate the MemPurge prototype
// (values >1.0 are equivalent to 1.0).
options . experimental_mempurge_threshold = 2.5 ;
ASSERT_OK ( TryReopen ( options ) ) ;
Baptiste Lemaire
2 years ago
committed by