@ -632,6 +632,7 @@ class BackupableDBTest : public testing::Test {
options_ . write_buffer_size = 1 < < 17 ; // 128KB
options_ . write_buffer_size = 1 < < 17 ; // 128KB
options_ . env = test_db_env_ . get ( ) ;
options_ . env = test_db_env_ . get ( ) ;
options_ . wal_dir = dbname_ ;
options_ . wal_dir = dbname_ ;
options_ . enable_blob_files = true ;
// Create logger
// Create logger
DBOptions logger_options ;
DBOptions logger_options ;
@ -894,15 +895,18 @@ class BackupableDBTest : public testing::Test {
void AssertDirectoryFilesMatchRegex ( const std : : string & dir ,
void AssertDirectoryFilesMatchRegex ( const std : : string & dir ,
const std : : regex & pattern ,
const std : : regex & pattern ,
const std : : string & file_type ,
int minimum_count ) {
int minimum_count ) {
std : : vector < FileAttributes > children ;
std : : vector < FileAttributes > children ;
ASSERT_OK ( file_manager_ - > GetChildrenFileAttributes ( dir , & children ) ) ;
ASSERT_OK ( file_manager_ - > GetChildrenFileAttributes ( dir , & children ) ) ;
int found_count = 0 ;
int found_count = 0 ;
for ( const auto & child : children ) {
for ( const auto & child : children ) {
const std : : string match ( " match " ) ;
if ( EndsWith ( child . name , file_type ) ) {
ASSERT_EQ ( match , std : : regex_replace ( child . name , pattern , match ) ) ;
ASSERT_TRUE ( std : : regex_match ( child . name , pattern ) )
< < " File name " < < child . name < < " does not match regex. " ;
+ + found_count ;
+ + found_count ;
}
}
}
ASSERT_GE ( found_count , minimum_count ) ;
ASSERT_GE ( found_count , minimum_count ) ;
}
}
@ -1433,9 +1437,8 @@ TEST_F(BackupableDBTest, CorruptFileMaintainSize) {
}
}
// Corrupt a blob file but maintain its size
// Corrupt a blob file but maintain its size
TEST_F ( BackupableDBTest , CorruptBlobFileMaintainSize ) {
TEST_P ( BackupableDBTestWithParam , CorruptBlobFileMaintainSize ) {
const int keys_iteration = 5000 ;
const int keys_iteration = 5000 ;
options_ . enable_blob_files = true ;
OpenDBAndBackupEngine ( true ) ;
OpenDBAndBackupEngine ( true ) ;
// create a backup
// create a backup
FillDB ( db_ . get ( ) , 0 , keys_iteration ) ;
FillDB ( db_ . get ( ) , 0 , keys_iteration ) ;
@ -1450,12 +1453,18 @@ TEST_F(BackupableDBTest, CorruptBlobFileMaintainSize) {
std : : string file_to_corrupt ;
std : : string file_to_corrupt ;
std : : vector < FileAttributes > children ;
std : : vector < FileAttributes > children ;
const std : : string dir = backupdir_ + " /private/1 " ;
std : : string dir = backupdir_ ;
if ( backupable_options_ - > share_files_with_checksum ) {
dir + = " /shared_checksum " ;
} else {
dir + = " /shared " ;
}
ASSERT_OK ( file_manager_ - > GetChildrenFileAttributes ( dir , & children ) ) ;
ASSERT_OK ( file_manager_ - > GetChildrenFileAttributes ( dir , & children ) ) ;
for ( const auto & child : children ) {
for ( const auto & child : children ) {
if ( child . name . find ( " .blob " ) ! = std : : string : : npos & &
if ( EndsWith ( child . name , " .blob " ) & & child . size_bytes ! = 0 ) {
child . size_bytes ! = 0 ) {
// corrupt the blob files by replacing its content by file_size random
// corrupt the blob files by replacing its content by file_size random
// bytes
// bytes
ASSERT_OK (
ASSERT_OK (
@ -1509,7 +1518,6 @@ TEST_F(BackupableDBTest, TableFileCorruptedBeforeBackup) {
// been corrupted and the blob file checksum is stored in the DB manifest
// been corrupted and the blob file checksum is stored in the DB manifest
TEST_F ( BackupableDBTest , BlobFileCorruptedBeforeBackup ) {
TEST_F ( BackupableDBTest , BlobFileCorruptedBeforeBackup ) {
const int keys_iteration = 50000 ;
const int keys_iteration = 50000 ;
options_ . enable_blob_files = true ;
OpenDBAndBackupEngine ( true /* destroy_old_data */ , false /* dummy */ ,
OpenDBAndBackupEngine ( true /* destroy_old_data */ , false /* dummy */ ,
kNoShare ) ;
kNoShare ) ;
@ -1575,7 +1583,6 @@ TEST_P(BackupableDBTestWithParam, TableFileCorruptedBeforeBackup) {
// the case when backup blob files will be stored in a shared directory
// the case when backup blob files will be stored in a shared directory
TEST_P ( BackupableDBTestWithParam , BlobFileCorruptedBeforeBackup ) {
TEST_P ( BackupableDBTestWithParam , BlobFileCorruptedBeforeBackup ) {
const int keys_iteration = 50000 ;
const int keys_iteration = 50000 ;
options_ . enable_blob_files = true ;
OpenDBAndBackupEngine ( true /* destroy_old_data */ ) ;
OpenDBAndBackupEngine ( true /* destroy_old_data */ ) ;
FillDB ( db_ . get ( ) , 0 , keys_iteration ) ;
FillDB ( db_ . get ( ) , 0 , keys_iteration ) ;
CloseAndReopenDB ( /*read_only*/ true ) ;
CloseAndReopenDB ( /*read_only*/ true ) ;
@ -1986,6 +1993,8 @@ TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsNewNaming) {
" [0-9]+_s[0-9A-Z]{20}_[0-9]+[.]sst " } ,
" [0-9]+_s[0-9A-Z]{20}_[0-9]+[.]sst " } ,
} ;
} ;
const std : : string blobfile_pattern = " [0-9]+_[0-9]+_[0-9]+[.]blob " ;
for ( const auto & pair : option_to_expected ) {
for ( const auto & pair : option_to_expected ) {
CloseAndReopenDB ( ) ;
CloseAndReopenDB ( ) ;
backupable_options_ - > share_files_with_checksum_naming = pair . first ;
backupable_options_ - > share_files_with_checksum_naming = pair . first ;
@ -1994,12 +2003,15 @@ TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsNewNaming) {
CloseDBAndBackupEngine ( ) ;
CloseDBAndBackupEngine ( ) ;
AssertBackupConsistency ( 1 , 0 , keys_iteration , keys_iteration * 2 ) ;
AssertBackupConsistency ( 1 , 0 , keys_iteration , keys_iteration * 2 ) ;
AssertDirectoryFilesMatchRegex ( backupdir_ + " /shared_checksum " ,
AssertDirectoryFilesMatchRegex ( backupdir_ + " /shared_checksum " ,
std : : regex ( pair . second ) ,
std : : regex ( pair . second ) , " .sst " ,
1 /* minimum_count */ ) ;
1 /* minimum_count */ ) ;
if ( std : : string : : npos ! = pair . second . find ( " _[0-9]+[.]sst " ) ) {
if ( std : : string : : npos ! = pair . second . find ( " _[0-9]+[.]sst " ) ) {
AssertDirectoryFilesSizeIndicators ( backupdir_ + " /shared_checksum " ,
AssertDirectoryFilesSizeIndicators ( backupdir_ + " /shared_checksum " ,
1 /* minimum_count */ ) ;
1 /* minimum_count */ ) ;
}
}
AssertDirectoryFilesMatchRegex ( backupdir_ + " /shared_checksum " ,
std : : regex ( blobfile_pattern ) , " .blob " ,
1 /* minimum_count */ ) ;
}
}
}
}
@ -2024,6 +2036,8 @@ TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsOldFileNaming) {
// Old names should always be used on old files
// Old names should always be used on old files
const std : : regex expected ( " [0-9]+_[0-9]+_[0-9]+[.]sst " ) ;
const std : : regex expected ( " [0-9]+_[0-9]+_[0-9]+[.]sst " ) ;
const std : : string blobfile_pattern = " [0-9]+_[0-9]+_[0-9]+[.]blob " ;
for ( ShareFilesNaming option : { kNamingDefault , kUseDbSessionId } ) {
for ( ShareFilesNaming option : { kNamingDefault , kUseDbSessionId } ) {
CloseAndReopenDB ( ) ;
CloseAndReopenDB ( ) ;
backupable_options_ - > share_files_with_checksum_naming = option ;
backupable_options_ - > share_files_with_checksum_naming = option ;
@ -2032,6 +2046,9 @@ TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsOldFileNaming) {
CloseDBAndBackupEngine ( ) ;
CloseDBAndBackupEngine ( ) ;
AssertBackupConsistency ( 1 , 0 , keys_iteration , keys_iteration * 2 ) ;
AssertBackupConsistency ( 1 , 0 , keys_iteration , keys_iteration * 2 ) ;
AssertDirectoryFilesMatchRegex ( backupdir_ + " /shared_checksum " , expected ,
AssertDirectoryFilesMatchRegex ( backupdir_ + " /shared_checksum " , expected ,
" .sst " , 1 /* minimum_count */ ) ;
AssertDirectoryFilesMatchRegex ( backupdir_ + " /shared_checksum " ,
std : : regex ( blobfile_pattern ) , " .blob " ,
1 /* minimum_count */ ) ;
1 /* minimum_count */ ) ;
}
}
@ -2175,9 +2192,9 @@ TEST_F(BackupableDBTest, FileSizeForIncremental) {
ASSERT_OK ( backup_engine_ - > CreateNewBackup ( db_ . get ( ) , true /*flush*/ ) ) ;
ASSERT_OK ( backup_engine_ - > CreateNewBackup ( db_ . get ( ) , true /*flush*/ ) ) ;
CloseDBAndBackupEngine ( ) ;
CloseDBAndBackupEngine ( ) ;
// Corrupt backup SST
// Corrupt backup SST and blob file
ASSERT_OK ( file_manager_ - > GetChildrenFileAttributes ( shared_dir , & children ) ) ;
ASSERT_OK ( file_manager_ - > GetChildrenFileAttributes ( shared_dir , & children ) ) ;
ASSERT_EQ ( children . size ( ) , 1 U) ; // one sst
ASSERT_EQ ( children . size ( ) , 2 U) ; // one sst and one blob file
for ( const auto & child : children ) {
for ( const auto & child : children ) {
if ( child . name . size ( ) > 4 & & child . size_bytes > 0 ) {
if ( child . name . size ( ) > 4 & & child . size_bytes > 0 ) {
ASSERT_OK (
ASSERT_OK (
@ -2234,10 +2251,10 @@ TEST_F(BackupableDBTest, FileSizeForIncremental) {
OpenDBAndBackupEngine ( false , false , share ) ;
OpenDBAndBackupEngine ( false , false , share ) ;
ASSERT_OK ( db_ - > Put ( WriteOptions ( ) , " y " , Random ( 42 ) . RandomString ( 500 ) ) ) ;
ASSERT_OK ( db_ - > Put ( WriteOptions ( ) , " y " , Random ( 42 ) . RandomString ( 500 ) ) ) ;
// Count backup SSTs
// Count backup SSTs and blob files.
children . clear ( ) ;
children . clear ( ) ;
ASSERT_OK ( file_manager_ - > GetChildrenFileAttributes ( shared_dir , & children ) ) ;
ASSERT_OK ( file_manager_ - > GetChildrenFileAttributes ( shared_dir , & children ) ) ;
ASSERT_EQ ( children . size ( ) , 2 U) ; // two sst
ASSERT_EQ ( children . size ( ) , 4 U) ; // two sst and two blob files
// Try create backup 3
// Try create backup 3
s = backup_engine_ - > CreateNewBackup ( db_ . get ( ) , true /*flush*/ ) ;
s = backup_engine_ - > CreateNewBackup ( db_ . get ( ) , true /*flush*/ ) ;
@ -2250,18 +2267,18 @@ TEST_F(BackupableDBTest, FileSizeForIncremental) {
// Acceptable to call it corruption if size is not in name and
// Acceptable to call it corruption if size is not in name and
// db session id collision is practically impossible.
// db session id collision is practically impossible.
EXPECT_TRUE ( s . IsCorruption ( ) ) ;
EXPECT_TRUE ( s . IsCorruption ( ) ) ;
EXPECT_EQ ( children . size ( ) , 2 U) ; // no SST added
EXPECT_EQ ( children . size ( ) , 4 U) ; // no SST/Blob file added
} else if ( option = = share_no_checksum ) {
} else if ( option = = share_no_checksum ) {
// Good to call it corruption if both backups cannot be
// Good to call it corruption if both backups cannot be
// accommodated.
// accommodated.
EXPECT_TRUE ( s . IsCorruption ( ) ) ;
EXPECT_TRUE ( s . IsCorruption ( ) ) ;
EXPECT_EQ ( children . size ( ) , 2 U) ; // no SST added
EXPECT_EQ ( children . size ( ) , 4 U) ; // no SST/Blob file added
} else {
} else {
// Since opening a DB seems sufficient for detecting size corruption
// Since opening a DB seems sufficient for detecting size corruption
// on the DB side, this should be a good thing, ...
// on the DB side, this should be a good thing, ...
EXPECT_OK ( s ) ;
EXPECT_OK ( s ) ;
// ... as long as we did actually treat it as a distinct SST file.
// ... as long as we did actually treat it as a distinct SST file.
EXPECT_EQ ( children . size ( ) , 3 U) ; // Another SST added
EXPECT_EQ ( children . size ( ) , 6 U) ; // Another SST and blob added
}
}
CloseDBAndBackupEngine ( ) ;
CloseDBAndBackupEngine ( ) ;
ASSERT_OK ( DestroyDB ( dbname_ , options_ ) ) ;
ASSERT_OK ( DestroyDB ( dbname_ , options_ ) ) ;