NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Isolate db env and backup Env in unit tests

Browse Source 
Summary:
- Used ChrootEnv so the database and backup Envs are isolated in the filesystem.
- Removed DifferentEnvs test since now every test uses different Envs

Depends on D57543

Test Plan:
- ran backupable_db_test
- verified backupable_db_test now catches the bug when D57159 is backed out (this bug previously passed through the test cases, which motivated this change)

Reviewers: sdong, lightmark, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D57615
main
Andrew Kryczka 9 years ago
parent 560358dc93
commit e61ba052b3
1 changed files with 56 additions and 86 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 142
      utilities/backupable/backupable_db_test.cc

142
utilities/backupable/backupable_db_test.cc
Unescape View File

@ -9,25 +9,24 @@
#ifndef ROCKSDB_LITE
#include <string>
#include <algorithm>
#include <iostream>
#include <string>
#include "db/db_impl.h"
#include "db/filename.h"
#include "port/port.h"
#include "port/stack_trace.h"
#include "rocksdb/types.h"
#include "rocksdb/transaction_log.h"
#include "rocksdb/types.h"
#include "rocksdb/utilities/backupable_db.h"
#include "util/env_chroot.h"
#include "util/file_reader_writer.h"
#include "util/testharness.h"
#include "util/random.h"
#include "util/mutexlock.h"
#include "util/random.h"
#include "util/string_util.h"
#include "util/sync_point.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "util/mock_env.h"
namespace rocksdb {
@ -428,15 +427,19 @@ class BackupableDBTest : public testing::Test {
public:
BackupableDBTest() {
// set up files
dbname_ = test::TmpDir() + "/backupable_db";
backupdir_ = test::TmpDir() + "/backupable_db_backup";
std::string db_chroot = test::TmpDir() + "/backupable_db";
std::string backup_chroot = test::TmpDir() + "/backupable_db_backup";
Env::Default()->CreateDir(db_chroot);
Env::Default()->CreateDir(backup_chroot);
dbname_ = "/tempdb";
backupdir_ = "/tempbk";
// set up envs
env_ = Env::Default();
mock_env_.reset(new MockEnv(env_));
test_db_env_.reset(new TestEnv(env_));
test_backup_env_.reset(new TestEnv(env_));
file_manager_.reset(new FileManager(env_));
db_chroot_env_.reset(NewChrootEnv(Env::Default(), db_chroot));
backup_chroot_env_.reset(NewChrootEnv(Env::Default(), backup_chroot));
test_db_env_.reset(new TestEnv(db_chroot_env_.get()));
test_backup_env_.reset(new TestEnv(backup_chroot_env_.get()));
file_manager_.reset(new FileManager(backup_chroot_env_.get()));
// set up db options
options_.create_if_missing = true;
@ -447,8 +450,7 @@ class BackupableDBTest : public testing::Test {
// Create logger
DBOptions logger_options;
logger_options.env = env_;
logger_options.db_log_dir = backupdir_;
logger_options.env = db_chroot_env_.get();
CreateLoggerFromOptions(dbname_, logger_options, &logger_);
// set up backup db options
@ -459,7 +461,7 @@ class BackupableDBTest : public testing::Test {
backupable_options_->max_background_operations = 7;
// delete old files in db
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
DB* OpenDB() {
@ -548,13 +550,13 @@ class BackupableDBTest : public testing::Test {
void DeleteLogFiles() {
std::vector<std::string> delete_logs;
env_->GetChildren(dbname_, &delete_logs);
db_chroot_env_->GetChildren(dbname_, &delete_logs);
for (auto f : delete_logs) {
uint64_t number;
FileType type;
bool ok = ParseFileName(f, &number, &type);
if (ok && type == kLogFile) {
env_->DeleteFile(dbname_ + "/" + f);
db_chroot_env_->DeleteFile(dbname_ + "/" + f);
}
}
}
@ -568,8 +570,8 @@ class BackupableDBTest : public testing::Test {
std::shared_ptr<Logger> logger_;
// envs
Env* env_;
unique_ptr<MockEnv> mock_env_;
unique_ptr<Env> db_chroot_env_;
unique_ptr<Env> backup_chroot_env_;
unique_ptr<TestEnv> test_db_env_;
unique_ptr<TestEnv> test_backup_env_;
unique_ptr<FileManager> file_manager_;
@ -642,7 +644,7 @@ TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
// second iter -- don't flush before backup
for (int iter = 0; iter < 2; ++iter) {
// delete old data
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
bool destroy_data = true;
// every iteration --
@ -659,7 +661,7 @@ TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
FillDB(db_.get(), keys_iteration * i, fill_up_to);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0));
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
// ---- make sure it's empty ----
DB* db = OpenDB();
@ -687,7 +689,7 @@ TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
const int max_key = keys_iteration * 4 + 10;
Random rnd(7);
// delete old data
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true);
// write some data, backup, repeat
@ -706,7 +708,7 @@ TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
}
// close and destroy
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
// ---- make sure it's empty ----
DB* db = OpenDB();
@ -763,7 +765,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
"/private/1.tmp/CURRENT", "/private/1.tmp/MANIFEST-01",
"/private/1.tmp/00011.log", "/meta/1.tmp",
"/LATEST_BACKUP.tmp"};
AppendPath(dbname_ + "_backup", should_have_written);
AppendPath(backupdir_, should_have_written);
test_backup_env_->AssertWrittenFiles(should_have_written);
// should write 4 new DB files + LATEST_BACKUP + one meta file
@ -784,7 +786,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
"/meta/2.tmp",
"/LATEST_BACKUP.tmp"
};
AppendPath(dbname_ + "_backup", should_have_written);
AppendPath(backupdir_, should_have_written);
test_backup_env_->AssertWrittenFiles(should_have_written);
ASSERT_OK(backup_engine_->DeleteBackup(1));
@ -805,41 +807,6 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
CloseDBAndBackupEngine();
}
// Verify that backup works when the database environment is not the same as
// the backup environment
// TODO(agf): Make all/most tests use different db and backup environments.
// This will probably require more implementation of MockEnv.
// For example, MockEnv::RenameFile() must be able to rename
// directories.
TEST_F(BackupableDBTest, DifferentEnvs) {
test_db_env_.reset(new TestEnv(mock_env_.get()));
options_.env = test_db_env_.get();
OpenDBAndBackupEngine(true, true);
// should write 5 DB files + LATEST_BACKUP + one meta file
test_backup_env_->SetLimitWrittenFiles(7);
test_backup_env_->ClearWrittenFiles();
test_db_env_->SetLimitWrittenFiles(0);
dummy_db_->live_files_ = { "/00010.sst", "/00011.sst",
"/CURRENT", "/MANIFEST-01" };
dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
CloseDBAndBackupEngine();
// try simple backup and verify correctness
OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
test_db_env_->SetFilenamesForMockedAttrs({});
AssertBackupConsistency(0, 0, 100, 500);
}
// test various kind of corruptions that may happen:
// 1. Not able to write a file for backup - that backup should fail,
// everything else should work
@ -893,7 +860,7 @@ TEST_F(BackupableDBTest, CorruptionsTest) {
// assert that we wrote 6 to LATEST_BACKUP
{
std::string latest_backup_contents;
ReadFileToString(env_, backupdir_ + "/LATEST_BACKUP",
ReadFileToString(backup_chroot_env_.get(), backupdir_ + "/LATEST_BACKUP",
&latest_backup_contents);
ASSERT_EQ(std::atol(latest_backup_contents.c_str()), 6);
}
@ -993,7 +960,8 @@ TEST_F(BackupableDBTest, NoDeleteWithReadOnly) {
backupable_options_->destroy_old_data = false;
BackupEngineReadOnly* read_only_backup_engine;
ASSERT_OK(BackupEngineReadOnly::Open(env_, *backupable_options_,
ASSERT_OK(BackupEngineReadOnly::Open(backup_chroot_env_.get(),
*backupable_options_,
&read_only_backup_engine));
// assert that data from backup 5 is still here (even though LATEST_BACKUP
@ -1160,7 +1128,7 @@ TEST_F(BackupableDBTest, RateLimiting) {
for (const auto& limit : limits) {
// destroy old data
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
backupable_options_->backup_rate_limit = limit.first;
backupable_options_->restore_rate_limit = limit.second;
@ -1169,9 +1137,9 @@ TEST_F(BackupableDBTest, RateLimiting) {
OpenDBAndBackupEngine(true);
size_t bytes_written = FillDB(db_.get(), 0, 100000);
auto start_backup = env_->NowMicros();
auto start_backup = db_chroot_env_->NowMicros();
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
auto backup_time = env_->NowMicros() - start_backup;
auto backup_time = db_chroot_env_->NowMicros() - start_backup;
auto rate_limited_backup_time = (bytes_written * kMicrosPerSec) /
backupable_options_->backup_rate_limit;
ASSERT_GT(backup_time, 0.8 * rate_limited_backup_time);
@ -1179,9 +1147,9 @@ TEST_F(BackupableDBTest, RateLimiting) {
CloseDBAndBackupEngine();
OpenBackupEngine();
auto start_restore = env_->NowMicros();
auto start_restore = db_chroot_env_->NowMicros();
ASSERT_OK(backup_engine_->RestoreDBFromLatestBackup(dbname_, dbname_));
auto restore_time = env_->NowMicros() - start_restore;
auto restore_time = db_chroot_env_->NowMicros() - start_restore;
CloseBackupEngine();
auto rate_limited_restore_time = (bytes_written * kMicrosPerSec) /
backupable_options_->restore_rate_limit;
@ -1193,21 +1161,21 @@ TEST_F(BackupableDBTest, RateLimiting) {
}
TEST_F(BackupableDBTest, ReadOnlyBackupEngine) {
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
FillDB(db_.get(), 100, 200);
ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
backupable_options_->destroy_old_data = false;
test_backup_env_->ClearWrittenFiles();
test_backup_env_->SetLimitDeleteFiles(0);
BackupEngineReadOnly* read_only_backup_engine;
ASSERT_OK(BackupEngineReadOnly::Open(env_, *backupable_options_,
&read_only_backup_engine));
ASSERT_OK(BackupEngineReadOnly::Open(
db_chroot_env_.get(), *backupable_options_, &read_only_backup_engine));
std::vector<BackupInfo> backup_info;
read_only_backup_engine->GetBackupInfo(&backup_info);
ASSERT_EQ(backup_info.size(), 2U);
@ -1225,7 +1193,7 @@ TEST_F(BackupableDBTest, ReadOnlyBackupEngine) {
}
TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) {
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100);
bool is_callback_invoked = false;
@ -1235,14 +1203,14 @@ TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) {
ASSERT_TRUE(is_callback_invoked);
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
OpenDBAndBackupEngine(true);
env_->CreateDirIfMissing(backupdir_ + "/shared");
backup_chroot_env_->CreateDirIfMissing(backupdir_ + "/shared");
std::string file_five = backupdir_ + "/shared/000007.sst";
std::string file_five_contents = "I'm not really a sst file";
// this depends on the fact that 00007.sst is the first file created by the DB
@ -1253,7 +1221,8 @@ TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok());
std::string new_file_five_contents;
ASSERT_OK(ReadFileToString(env_, file_five, &new_file_five_contents));
ASSERT_OK(ReadFileToString(backup_chroot_env_.get(), file_five,
&new_file_five_contents));
// file 000007.sst was overwritten
ASSERT_TRUE(new_file_five_contents != file_five_contents);
@ -1301,7 +1270,7 @@ TEST_F(BackupableDBTest, EnvFailures) {
// Verify manifest can roll while a backup is being created with the old
// manifest.
TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
options_.max_manifest_file_size = 0; // always rollover manifest for file add
OpenDBAndBackupEngine(true);
FillDB(db_.get(), 0, 100);
@ -1328,12 +1297,12 @@ TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
std::string prev_manifest_path =
DescriptorFileName(dbname_, db_impl->TEST_Current_Manifest_FileNo());
FillDB(db_.get(), 0, 100);
ASSERT_OK(env_->FileExists(prev_manifest_path));
ASSERT_OK(db_chroot_env_->FileExists(prev_manifest_path));
ASSERT_OK(db_->Flush(FlushOptions()));
ASSERT_TRUE(env_->FileExists(prev_manifest_path).IsNotFound());
ASSERT_TRUE(db_chroot_env_->FileExists(prev_manifest_path).IsNotFound());
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
AssertBackupConsistency(0, 0, 100);
}
@ -1341,9 +1310,10 @@ TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
TEST_F(BackupableDBTest, Issue921Test) {
BackupEngine* backup_engine;
backupable_options_->share_table_files = false;
env_->CreateDirIfMissing(backupable_options_->backup_dir);
backup_chroot_env_->CreateDirIfMissing(backupable_options_->backup_dir);
backupable_options_->backup_dir += "/new_dir";
ASSERT_OK(BackupEngine::Open(env_, *backupable_options_, &backup_engine));
ASSERT_OK(BackupEngine::Open(backup_chroot_env_.get(), *backupable_options_,
&backup_engine));
delete backup_engine;
}
@ -1368,7 +1338,7 @@ TEST_F(BackupableDBTest, BackupWithMetadata) {
ASSERT_EQ(std::to_string(i), backup_infos[i].app_metadata);
}
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
TEST_F(BackupableDBTest, BinaryMetadata) {
@ -1386,7 +1356,7 @@ TEST_F(BackupableDBTest, BinaryMetadata) {
ASSERT_EQ(1, backup_infos.size());
ASSERT_EQ(binaryMetadata, backup_infos[0].app_metadata);
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
TEST_F(BackupableDBTest, MetadataTooLarge) {
@ -1395,7 +1365,7 @@ TEST_F(BackupableDBTest, MetadataTooLarge) {
ASSERT_NOK(
backup_engine_->CreateNewBackupWithMetadata(db_.get(), largeMetadata));
CloseDBAndBackupEngine();
DestroyDB(dbname_, Options());
DestroyDB(dbname_, options_);
}
} // anon namespace

Write Preview
Loading…
Cancel
Save