Isolate db env and backup Env in unit tests

Summary:
- Used ChrootEnv so the database and backup Envs are isolated in the filesystem.
- Removed DifferentEnvs test since now every test uses different Envs

Depends on D57543

Test Plan:
- ran backupable_db_test
- verified backupable_db_test now catches the bug when D57159 is backed out (this bug previously passed through the test cases, which motivated this change)

Reviewers: sdong, lightmark, IslamAbdelRahman

Reviewed By: IslamAbdelRahman

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D57615

utilities/backupable/backupable_db_test.cc

@@ -9,25 +9,24 @@
 
 #ifndef ROCKSDB_LITE
 
-#include <string>
 #include <algorithm>
-#include <iostream>
+#include <string>
 
 #include "db/db_impl.h"
 #include "db/filename.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
-#include "rocksdb/types.h"
 #include "rocksdb/transaction_log.h"
+#include "rocksdb/types.h"
 #include "rocksdb/utilities/backupable_db.h"
+#include "util/env_chroot.h"
 #include "util/file_reader_writer.h"
-#include "util/testharness.h"
-#include "util/random.h"
 #include "util/mutexlock.h"
+#include "util/random.h"
 #include "util/string_util.h"
 #include "util/sync_point.h"
+#include "util/testharness.h"
 #include "util/testutil.h"
-#include "util/mock_env.h"
 
 namespace rocksdb {
 
@@ -428,15 +427,19 @@ class BackupableDBTest : public testing::Test {
  public:
   BackupableDBTest() {
     // set up files
-    dbname_ = test::TmpDir() + "/backupable_db";
+    std::string db_chroot = test::TmpDir() + "/backupable_db";
-    backupdir_ = test::TmpDir() + "/backupable_db_backup";
+    std::string backup_chroot = test::TmpDir() + "/backupable_db_backup";
+    Env::Default()->CreateDir(db_chroot);
+    Env::Default()->CreateDir(backup_chroot);
+    dbname_ = "/tempdb";
+    backupdir_ = "/tempbk";
 
     // set up envs
-    env_ = Env::Default();
+    db_chroot_env_.reset(NewChrootEnv(Env::Default(), db_chroot));
-    mock_env_.reset(new MockEnv(env_));
+    backup_chroot_env_.reset(NewChrootEnv(Env::Default(), backup_chroot));
-    test_db_env_.reset(new TestEnv(env_));
+    test_db_env_.reset(new TestEnv(db_chroot_env_.get()));
-    test_backup_env_.reset(new TestEnv(env_));
+    test_backup_env_.reset(new TestEnv(backup_chroot_env_.get()));
-    file_manager_.reset(new FileManager(env_));
+    file_manager_.reset(new FileManager(backup_chroot_env_.get()));
 
     // set up db options
     options_.create_if_missing = true;
@@ -447,8 +450,7 @@ class BackupableDBTest : public testing::Test {
 
     // Create logger
     DBOptions logger_options;
-    logger_options.env = env_;
+    logger_options.env = db_chroot_env_.get();
-    logger_options.db_log_dir = backupdir_;
     CreateLoggerFromOptions(dbname_, logger_options, &logger_);
 
     // set up backup db options
@@ -459,7 +461,7 @@ class BackupableDBTest : public testing::Test {
     backupable_options_->max_background_operations = 7;
 
     // delete old files in db
-    DestroyDB(dbname_, Options());
+    DestroyDB(dbname_, options_);
   }
 
   DB* OpenDB() {
@@ -548,13 +550,13 @@ class BackupableDBTest : public testing::Test {
 
   void DeleteLogFiles() {
     std::vector<std::string> delete_logs;
-    env_->GetChildren(dbname_, &delete_logs);
+    db_chroot_env_->GetChildren(dbname_, &delete_logs);
     for (auto f : delete_logs) {
       uint64_t number;
       FileType type;
       bool ok = ParseFileName(f, &number, &type);
       if (ok && type == kLogFile) {
-        env_->DeleteFile(dbname_ + "/" + f);
+        db_chroot_env_->DeleteFile(dbname_ + "/" + f);
       }
     }
   }
@@ -568,8 +570,8 @@ class BackupableDBTest : public testing::Test {
   std::shared_ptr<Logger> logger_;
 
   // envs
-  Env* env_;
+  unique_ptr<Env> db_chroot_env_;
-  unique_ptr<MockEnv> mock_env_;
+  unique_ptr<Env> backup_chroot_env_;
   unique_ptr<TestEnv> test_db_env_;
   unique_ptr<TestEnv> test_backup_env_;
   unique_ptr<FileManager> file_manager_;
@@ -642,7 +644,7 @@ TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
   // second iter -- don't flush before backup
   for (int iter = 0; iter < 2; ++iter) {
     // delete old data
-    DestroyDB(dbname_, Options());
+    DestroyDB(dbname_, options_);
     bool destroy_data = true;
 
     // every iteration --
@@ -659,7 +661,7 @@ TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) {
       FillDB(db_.get(), keys_iteration * i, fill_up_to);
       ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0));
       CloseDBAndBackupEngine();
-      DestroyDB(dbname_, Options());
+      DestroyDB(dbname_, options_);
 
       // ---- make sure it's empty ----
       DB* db = OpenDB();
@@ -687,7 +689,7 @@ TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
   const int max_key = keys_iteration * 4 + 10;
   Random rnd(7);
   // delete old data
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
 
   OpenDBAndBackupEngine(true);
   // write some data, backup, repeat
@@ -706,7 +708,7 @@ TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) {
   }
   // close and destroy
   CloseDBAndBackupEngine();
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
 
   // ---- make sure it's empty ----
   DB* db = OpenDB();
@@ -763,7 +765,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
       "/private/1.tmp/CURRENT",   "/private/1.tmp/MANIFEST-01",
       "/private/1.tmp/00011.log", "/meta/1.tmp",
       "/LATEST_BACKUP.tmp"};
-  AppendPath(dbname_ + "_backup", should_have_written);
+  AppendPath(backupdir_, should_have_written);
   test_backup_env_->AssertWrittenFiles(should_have_written);
 
   // should write 4 new DB files + LATEST_BACKUP + one meta file
@@ -784,7 +786,7 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
     "/meta/2.tmp",
     "/LATEST_BACKUP.tmp"
   };
-  AppendPath(dbname_ + "_backup", should_have_written);
+  AppendPath(backupdir_, should_have_written);
   test_backup_env_->AssertWrittenFiles(should_have_written);
 
   ASSERT_OK(backup_engine_->DeleteBackup(1));
@@ -805,41 +807,6 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
   CloseDBAndBackupEngine();
 }
 
-// Verify that backup works when the database environment is not the same as
-// the backup environment
-// TODO(agf): Make all/most tests use different db and backup environments.
-//            This will probably require more implementation of MockEnv.
-//            For example, MockEnv::RenameFile() must be able to rename
-//            directories.
-TEST_F(BackupableDBTest, DifferentEnvs) {
-  test_db_env_.reset(new TestEnv(mock_env_.get()));
-  options_.env = test_db_env_.get();
-
-  OpenDBAndBackupEngine(true, true);
-
-  // should write 5 DB files + LATEST_BACKUP + one meta file
-  test_backup_env_->SetLimitWrittenFiles(7);
-  test_backup_env_->ClearWrittenFiles();
-  test_db_env_->SetLimitWrittenFiles(0);
-  dummy_db_->live_files_ = { "/00010.sst", "/00011.sst",
-                             "/CURRENT",   "/MANIFEST-01" };
-  dummy_db_->wal_files_ = {{"/00011.log", true}, {"/00012.log", false}};
-  test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
-  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
-
-  CloseDBAndBackupEngine();
-
-  // try simple backup and verify correctness
-  OpenDBAndBackupEngine(true);
-  FillDB(db_.get(), 0, 100);
-  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
-  CloseDBAndBackupEngine();
-  DestroyDB(dbname_, Options());
-
-  test_db_env_->SetFilenamesForMockedAttrs({});
-  AssertBackupConsistency(0, 0, 100, 500);
-}
-
 // test various kind of corruptions that may happen:
 // 1. Not able to write a file for backup - that backup should fail,
 //      everything else should work
@@ -893,7 +860,7 @@ TEST_F(BackupableDBTest, CorruptionsTest) {
   // assert that we wrote 6 to LATEST_BACKUP
   {
     std::string latest_backup_contents;
-    ReadFileToString(env_, backupdir_ + "/LATEST_BACKUP",
+    ReadFileToString(backup_chroot_env_.get(), backupdir_ + "/LATEST_BACKUP",
                      &latest_backup_contents);
     ASSERT_EQ(std::atol(latest_backup_contents.c_str()), 6);
   }
@@ -993,7 +960,8 @@ TEST_F(BackupableDBTest, NoDeleteWithReadOnly) {
 
   backupable_options_->destroy_old_data = false;
   BackupEngineReadOnly* read_only_backup_engine;
-  ASSERT_OK(BackupEngineReadOnly::Open(env_, *backupable_options_,
+  ASSERT_OK(BackupEngineReadOnly::Open(backup_chroot_env_.get(),
+                                       *backupable_options_,
                                        &read_only_backup_engine));
 
   // assert that data from backup 5 is still here (even though LATEST_BACKUP
@@ -1160,7 +1128,7 @@ TEST_F(BackupableDBTest, RateLimiting) {
 
     for (const auto& limit : limits) {
       // destroy old data
-      DestroyDB(dbname_, Options());
+      DestroyDB(dbname_, options_);
 
       backupable_options_->backup_rate_limit = limit.first;
       backupable_options_->restore_rate_limit = limit.second;
@@ -1169,9 +1137,9 @@ TEST_F(BackupableDBTest, RateLimiting) {
       OpenDBAndBackupEngine(true);
       size_t bytes_written = FillDB(db_.get(), 0, 100000);
 
-      auto start_backup = env_->NowMicros();
+      auto start_backup = db_chroot_env_->NowMicros();
       ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
-      auto backup_time = env_->NowMicros() - start_backup;
+      auto backup_time = db_chroot_env_->NowMicros() - start_backup;
       auto rate_limited_backup_time = (bytes_written * kMicrosPerSec) /
                                       backupable_options_->backup_rate_limit;
       ASSERT_GT(backup_time, 0.8 * rate_limited_backup_time);
@@ -1179,9 +1147,9 @@ TEST_F(BackupableDBTest, RateLimiting) {
       CloseDBAndBackupEngine();
 
       OpenBackupEngine();
-      auto start_restore = env_->NowMicros();
+      auto start_restore = db_chroot_env_->NowMicros();
       ASSERT_OK(backup_engine_->RestoreDBFromLatestBackup(dbname_, dbname_));
-      auto restore_time = env_->NowMicros() - start_restore;
+      auto restore_time = db_chroot_env_->NowMicros() - start_restore;
       CloseBackupEngine();
       auto rate_limited_restore_time = (bytes_written * kMicrosPerSec) /
                                        backupable_options_->restore_rate_limit;
@@ -1193,21 +1161,21 @@ TEST_F(BackupableDBTest, RateLimiting) {
 }
 
 TEST_F(BackupableDBTest, ReadOnlyBackupEngine) {
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
   OpenDBAndBackupEngine(true);
   FillDB(db_.get(), 0, 100);
   ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
   FillDB(db_.get(), 100, 200);
   ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true));
   CloseDBAndBackupEngine();
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
 
   backupable_options_->destroy_old_data = false;
   test_backup_env_->ClearWrittenFiles();
   test_backup_env_->SetLimitDeleteFiles(0);
   BackupEngineReadOnly* read_only_backup_engine;
-  ASSERT_OK(BackupEngineReadOnly::Open(env_, *backupable_options_,
+  ASSERT_OK(BackupEngineReadOnly::Open(
-                                       &read_only_backup_engine));
+      db_chroot_env_.get(), *backupable_options_, &read_only_backup_engine));
   std::vector<BackupInfo> backup_info;
   read_only_backup_engine->GetBackupInfo(&backup_info);
   ASSERT_EQ(backup_info.size(), 2U);
@@ -1225,7 +1193,7 @@ TEST_F(BackupableDBTest, ReadOnlyBackupEngine) {
 }
 
 TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) {
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
   OpenDBAndBackupEngine(true);
   FillDB(db_.get(), 0, 100);
   bool is_callback_invoked = false;
@@ -1235,14 +1203,14 @@ TEST_F(BackupableDBTest, ProgressCallbackDuringBackup) {
 
   ASSERT_TRUE(is_callback_invoked);
   CloseDBAndBackupEngine();
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
 }
 
 TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
   OpenDBAndBackupEngine(true);
 
-  env_->CreateDirIfMissing(backupdir_ + "/shared");
+  backup_chroot_env_->CreateDirIfMissing(backupdir_ + "/shared");
   std::string file_five = backupdir_ + "/shared/000007.sst";
   std::string file_five_contents = "I'm not really a sst file";
   // this depends on the fact that 00007.sst is the first file created by the DB
@@ -1253,7 +1221,8 @@ TEST_F(BackupableDBTest, GarbageCollectionBeforeBackup) {
   ASSERT_TRUE(backup_engine_->CreateNewBackup(db_.get(), true).ok());
 
   std::string new_file_five_contents;
-  ASSERT_OK(ReadFileToString(env_, file_five, &new_file_five_contents));
+  ASSERT_OK(ReadFileToString(backup_chroot_env_.get(), file_five,
+                             &new_file_five_contents));
   // file 000007.sst was overwritten
   ASSERT_TRUE(new_file_five_contents != file_five_contents);
 
@@ -1301,7 +1270,7 @@ TEST_F(BackupableDBTest, EnvFailures) {
 // Verify manifest can roll while a backup is being created with the old
 // manifest.
 TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
   options_.max_manifest_file_size = 0;  // always rollover manifest for file add
   OpenDBAndBackupEngine(true);
   FillDB(db_.get(), 0, 100);
@@ -1328,12 +1297,12 @@ TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
   std::string prev_manifest_path =
       DescriptorFileName(dbname_, db_impl->TEST_Current_Manifest_FileNo());
   FillDB(db_.get(), 0, 100);
-  ASSERT_OK(env_->FileExists(prev_manifest_path));
+  ASSERT_OK(db_chroot_env_->FileExists(prev_manifest_path));
   ASSERT_OK(db_->Flush(FlushOptions()));
-  ASSERT_TRUE(env_->FileExists(prev_manifest_path).IsNotFound());
+  ASSERT_TRUE(db_chroot_env_->FileExists(prev_manifest_path).IsNotFound());
 
   CloseDBAndBackupEngine();
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
   AssertBackupConsistency(0, 0, 100);
 }
 
@@ -1341,9 +1310,10 @@ TEST_F(BackupableDBTest, ChangeManifestDuringBackupCreation) {
 TEST_F(BackupableDBTest, Issue921Test) {
   BackupEngine* backup_engine;
   backupable_options_->share_table_files = false;
-  env_->CreateDirIfMissing(backupable_options_->backup_dir);
+  backup_chroot_env_->CreateDirIfMissing(backupable_options_->backup_dir);
   backupable_options_->backup_dir += "/new_dir";
-  ASSERT_OK(BackupEngine::Open(env_, *backupable_options_, &backup_engine));
+  ASSERT_OK(BackupEngine::Open(backup_chroot_env_.get(), *backupable_options_,
+                               &backup_engine));
 
   delete backup_engine;
 }
@@ -1368,7 +1338,7 @@ TEST_F(BackupableDBTest, BackupWithMetadata) {
     ASSERT_EQ(std::to_string(i), backup_infos[i].app_metadata);
   }
   CloseDBAndBackupEngine();
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
 }
 
 TEST_F(BackupableDBTest, BinaryMetadata) {
@@ -1386,7 +1356,7 @@ TEST_F(BackupableDBTest, BinaryMetadata) {
   ASSERT_EQ(1, backup_infos.size());
   ASSERT_EQ(binaryMetadata, backup_infos[0].app_metadata);
   CloseDBAndBackupEngine();
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
 }
 
 TEST_F(BackupableDBTest, MetadataTooLarge) {
@@ -1395,7 +1365,7 @@ TEST_F(BackupableDBTest, MetadataTooLarge) {
   ASSERT_NOK(
       backup_engine_->CreateNewBackupWithMetadata(db_.get(), largeMetadata));
   CloseDBAndBackupEngine();
-  DestroyDB(dbname_, Options());
+  DestroyDB(dbname_, options_);
 }
 }  // anon namespace