Read-only BackupEngine

Summary: Read-only BackupEngine can connect to the same backup directory that is already running BackupEngine. That enables some interesting use-cases (i.e. restoring replica from primary's backup directory)

Test Plan: added a unit test

Reviewers: dhruba, haobo, ljin

Reviewed By: ljin

CC: leveldb

Differential Revision: https://reviews.facebook.net/D18297

include/utilities/backupable_db.h

@@ -117,6 +117,29 @@ struct BackupInfo {
       : backup_id(_backup_id), timestamp(_timestamp), size(_size) {}
 };
 
+class BackupEngineReadOnly {
+ public:
+  virtual ~BackupEngineReadOnly() {}
+
+  static BackupEngineReadOnly* NewReadOnlyBackupEngine(
+      Env* db_env, const BackupableDBOptions& options);
+
+  // You can GetBackupInfo safely, even with other BackupEngine performing
+  // backups on the same directory
+  virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info) = 0;
+
+  // Restoring DB from backup is NOT safe when there is another BackupEngine
+  // running that might call DeleteBackup() or PurgeOldBackups(). It is caller's
+  // responsibility to synchronize the operation, i.e. don't delete the backup
+  // when you're restoring from it
+  virtual Status RestoreDBFromBackup(
+      BackupID backup_id, const std::string& db_dir, const std::string& wal_dir,
+      const RestoreOptions& restore_options = RestoreOptions()) = 0;
+  virtual Status RestoreDBFromLatestBackup(
+      const std::string& db_dir, const std::string& wal_dir,
+      const RestoreOptions& restore_options = RestoreOptions()) = 0;
+};
+
 // Please see the documentation in BackupableDB and RestoreBackupableDB
 class BackupEngine {
  public:

utilities/backupable/backupable_db.cc

@@ -87,7 +87,8 @@ void BackupableDBOptions::Dump(Logger* logger) const {
 // -------- BackupEngineImpl class ---------
 class BackupEngineImpl : public BackupEngine {
  public:
-  BackupEngineImpl(Env* db_env, const BackupableDBOptions& options);
+  BackupEngineImpl(Env* db_env, const BackupableDBOptions& options,
+                   bool read_only = false);
   ~BackupEngineImpl();
   Status CreateNewBackup(DB* db, bool flush_before_backup = false);
   Status PurgeOldBackups(uint32_t num_backups_to_keep);
@@ -149,7 +150,7 @@ class BackupEngineImpl : public BackupEngine {
 
     Status AddFile(const FileInfo& file_info);
 
-    void Delete();
+    void Delete(bool delete_meta = true);
 
     bool Empty() {
       return files_.empty();
@@ -258,6 +259,7 @@ class BackupEngineImpl : public BackupEngine {
 
   static const size_t kDefaultCopyFileBufferSize = 5 * 1024 * 1024LL;  // 5MB
   size_t copy_file_buffer_size_;
+  bool read_only_;
 };
 
 BackupEngine* BackupEngine::NewBackupEngine(
@@ -266,27 +268,34 @@ BackupEngine* BackupEngine::NewBackupEngine(
 }
 
 BackupEngineImpl::BackupEngineImpl(Env* db_env,
-                                   const BackupableDBOptions& options)
+                                   const BackupableDBOptions& options,
+                                   bool read_only)
     : stop_backup_(false),
       options_(options),
       db_env_(db_env),
       backup_env_(options.backup_env != nullptr ? options.backup_env : db_env_),
-      copy_file_buffer_size_(kDefaultCopyFileBufferSize) {
+      copy_file_buffer_size_(kDefaultCopyFileBufferSize),
+      read_only_(read_only) {
+  if (read_only_) {
+    Log(options_.info_log, "Starting read_only backup engine");
+  }
   options_.Dump(options_.info_log);
 
-  // create all the dirs we need
+  if (!read_only_) {
-  backup_env_->CreateDirIfMissing(GetAbsolutePath());
+    // create all the dirs we need
-  backup_env_->NewDirectory(GetAbsolutePath(), &backup_directory_);
+    backup_env_->CreateDirIfMissing(GetAbsolutePath());
-  if (options_.share_table_files) {
+    backup_env_->NewDirectory(GetAbsolutePath(), &backup_directory_);
-    backup_env_->CreateDirIfMissing(GetAbsolutePath(GetSharedFileRel()));
+    if (options_.share_table_files) {
-    backup_env_->NewDirectory(GetAbsolutePath(GetSharedFileRel()),
+      backup_env_->CreateDirIfMissing(GetAbsolutePath(GetSharedFileRel()));
-                              &shared_directory_);
+      backup_env_->NewDirectory(GetAbsolutePath(GetSharedFileRel()),
+                                &shared_directory_);
+    }
+    backup_env_->CreateDirIfMissing(GetAbsolutePath(GetPrivateDirRel()));
+    backup_env_->NewDirectory(GetAbsolutePath(GetPrivateDirRel()),
+                              &private_directory_);
+    backup_env_->CreateDirIfMissing(GetBackupMetaDir());
+    backup_env_->NewDirectory(GetBackupMetaDir(), &meta_directory_);
   }
-  backup_env_->CreateDirIfMissing(GetAbsolutePath(GetPrivateDirRel()));
-  backup_env_->NewDirectory(GetAbsolutePath(GetPrivateDirRel()),
-                            &private_directory_);
-  backup_env_->CreateDirIfMissing(GetBackupMetaDir());
-  backup_env_->NewDirectory(GetBackupMetaDir(), &meta_directory_);
 
   std::vector<std::string> backup_meta_files;
   backup_env_->GetChildren(GetBackupMetaDir(), &backup_meta_files);
@@ -295,8 +304,10 @@ BackupEngineImpl::BackupEngineImpl(Env* db_env,
     BackupID backup_id = 0;
     sscanf(file.c_str(), "%u", &backup_id);
     if (backup_id == 0 || file != std::to_string(backup_id)) {
-      // invalid file name, delete that
+      if (!read_only_) {
-      backup_env_->DeleteFile(GetBackupMetaDir() + "/" + file);
+        // invalid file name, delete that
+        backup_env_->DeleteFile(GetBackupMetaDir() + "/" + file);
+      }
       continue;
     }
     assert(backups_.find(backup_id) == backups_.end());
@@ -306,6 +317,7 @@ BackupEngineImpl::BackupEngineImpl(Env* db_env,
   }
 
   if (options_.destroy_old_data) { // Destory old data
+    assert(!read_only_);
     for (auto& backup : backups_) {
       backup.second.Delete();
       obsolete_backups_.push_back(backup.first);
@@ -319,9 +331,12 @@ BackupEngineImpl::BackupEngineImpl(Env* db_env,
     for (auto& backup : backups_) {
       Status s = backup.second.LoadFromFile(options_.backup_dir);
       if (!s.ok()) {
-        Log(options_.info_log, "Backup %u corrupted - deleting -- %s",
+        Log(options_.info_log, "Backup %u corrupted -- %s", backup.first,
-            backup.first, s.ToString().c_str());
+            s.ToString().c_str());
-        backup.second.Delete();
+        if (!read_only_) {
+          Log(options_.info_log, "-> Deleting backup %u", backup.first);
+        }
+        backup.second.Delete(!read_only_);
         obsolete_backups_.push_back(backup.first);
       }
     }
@@ -331,6 +346,7 @@ BackupEngineImpl::BackupEngineImpl(Env* db_env,
     }
 
     Status s = GetLatestBackupFileContents(&latest_backup_id_);
+
     // If latest backup file is corrupted or non-existent
     // set latest backup as the biggest backup we have
     // or 0 if we have no backups
@@ -349,16 +365,18 @@ BackupEngineImpl::BackupEngineImpl(Env* db_env,
     itr = backups_.erase(itr);
   }
 
-  PutLatestBackupFileContents(latest_backup_id_); // Ignore errors
+  if (!read_only_) {
-  GarbageCollection(true);
+    PutLatestBackupFileContents(latest_backup_id_);  // Ignore errors
-  Log(options_.info_log,
+    GarbageCollection(true);
-      "Initialized BackupEngine, the latest backup is %u.",
+  }
+  Log(options_.info_log, "Initialized BackupEngine, the latest backup is %u.",
       latest_backup_id_);
 }
 
 BackupEngineImpl::~BackupEngineImpl() { LogFlush(options_.info_log); }
 
 Status BackupEngineImpl::CreateNewBackup(DB* db, bool flush_before_backup) {
+  assert(!read_only_);
   Status s;
   std::vector<std::string> live_files;
   VectorLogPtr live_wal_files;
@@ -499,6 +517,7 @@ Status BackupEngineImpl::CreateNewBackup(DB* db, bool flush_before_backup) {
 }
 
 Status BackupEngineImpl::PurgeOldBackups(uint32_t num_backups_to_keep) {
+  assert(!read_only_);
   Log(options_.info_log, "Purging old backups, keeping %u",
       num_backups_to_keep);
   while (num_backups_to_keep < backups_.size()) {
@@ -512,6 +531,7 @@ Status BackupEngineImpl::PurgeOldBackups(uint32_t num_backups_to_keep) {
 }
 
 Status BackupEngineImpl::DeleteBackup(BackupID backup_id) {
+  assert(!read_only_);
   Log(options_.info_log, "Deleting backup %u", backup_id);
   auto backup = backups_.find(backup_id);
   if (backup == backups_.end()) {
@@ -662,6 +682,7 @@ Status BackupEngineImpl::GetLatestBackupFileContents(uint32_t* latest_backup) {
 // do something like 1. delete file, 2. write new file
 // We write to a tmp file and then atomically rename
 Status BackupEngineImpl::PutLatestBackupFileContents(uint32_t latest_backup) {
+  assert(!read_only_);
   Status s;
   unique_ptr<WritableFile> file;
   EnvOptions env_options;
@@ -871,6 +892,7 @@ void BackupEngineImpl::DeleteChildren(const std::string& dir,
 }
 
 void BackupEngineImpl::GarbageCollection(bool full_scan) {
+  assert(!read_only_);
   Log(options_.info_log, "Starting garbage collection");
   std::vector<std::string> to_delete;
   for (auto& itr : backuped_file_infos_) {
@@ -973,7 +995,7 @@ Status BackupEngineImpl::BackupMeta::AddFile(const FileInfo& file_info) {
   return Status::OK();
 }
 
-void BackupEngineImpl::BackupMeta::Delete() {
+void BackupEngineImpl::BackupMeta::Delete(bool delete_meta) {
   for (const auto& file : files_) {
     auto itr = file_infos_->find(file);
     assert(itr != file_infos_->end());
@@ -981,7 +1003,9 @@ void BackupEngineImpl::BackupMeta::Delete() {
   }
   files_.clear();
   // delete meta file
-  env_->DeleteFile(meta_filename_);
+  if (delete_meta) {
+    env_->DeleteFile(meta_filename_);
+  }
   timestamp_ = 0;
 }
 
@@ -1107,6 +1131,45 @@ Status BackupEngineImpl::BackupMeta::StoreToFile(bool sync) {
   return s;
 }
 
+// -------- BackupEngineReadOnlyImpl ---------
+class BackupEngineReadOnlyImpl : public BackupEngineReadOnly {
+ public:
+  BackupEngineReadOnlyImpl(Env* db_env, const BackupableDBOptions& options) {
+    backup_engine_ = new BackupEngineImpl(db_env, options, true);
+  }
+  virtual ~BackupEngineReadOnlyImpl() {}
+
+  virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info) {
+    backup_engine_->GetBackupInfo(backup_info);
+  }
+
+  virtual Status RestoreDBFromBackup(
+      BackupID backup_id, const std::string& db_dir, const std::string& wal_dir,
+      const RestoreOptions& restore_options = RestoreOptions()) {
+    return backup_engine_->RestoreDBFromBackup(backup_id, db_dir, wal_dir,
+                                               restore_options);
+  }
+
+  virtual Status RestoreDBFromLatestBackup(
+      const std::string& db_dir, const std::string& wal_dir,
+      const RestoreOptions& restore_options = RestoreOptions()) {
+    return backup_engine_->RestoreDBFromLatestBackup(db_dir, wal_dir,
+                                                     restore_options);
+  }
+
+ private:
+  BackupEngineImpl* backup_engine_;
+};
+
+BackupEngineReadOnly* BackupEngineReadOnly::NewReadOnlyBackupEngine(
+    Env* db_env, const BackupableDBOptions& options) {
+  if (options.destroy_old_data) {
+    assert(false);
+    return nullptr;
+  }
+  return new BackupEngineReadOnlyImpl(db_env, options);
+}
+
 // --- BackupableDB methods --------
 
 BackupableDB::BackupableDB(DB* db, const BackupableDBOptions& options)

utilities/backupable/backupable_db_test.cc

@@ -178,6 +178,12 @@ class TestEnv : public EnvWrapper {
     return EnvWrapper::NewWritableFile(f, r, options);
   }
 
+  virtual Status DeleteFile(const std::string& fname) override {
+    ASSERT_GT(limit_delete_files_, 0);
+    limit_delete_files_--;
+    return EnvWrapper::DeleteFile(fname);
+  }
+
   void AssertWrittenFiles(std::vector<std::string>& should_have_written) {
     sort(should_have_written.begin(), should_have_written.end());
     sort(written_files_.begin(), written_files_.end());
@@ -192,6 +198,8 @@ class TestEnv : public EnvWrapper {
     limit_written_files_ = limit;
   }
 
+  void SetLimitDeleteFiles(uint64_t limit) { limit_delete_files_ = limit; }
+
   void SetDummySequentialFile(bool dummy_sequential_file) {
     dummy_sequential_file_ = dummy_sequential_file;
   }
@@ -200,7 +208,8 @@ class TestEnv : public EnvWrapper {
   bool dummy_sequential_file_ = false;
   std::vector<std::string> written_files_;
   uint64_t limit_written_files_ = 1000000;
-}; // TestEnv
+  uint64_t limit_delete_files_ = 1000000;
+};  // TestEnv
 
 class FileManager : public EnvWrapper {
  public:
@@ -864,7 +873,38 @@ TEST(BackupableDBTest, RateLimiting) {
   }
 }
 
-} // anon namespace
+TEST(BackupableDBTest, ReadOnlyBackupEngine) {
+  DestroyDB(dbname_, Options());
+  OpenBackupableDB(true);
+  FillDB(db_.get(), 0, 100);
+  ASSERT_OK(db_->CreateNewBackup(true));
+  FillDB(db_.get(), 100, 200);
+  ASSERT_OK(db_->CreateNewBackup(true));
+  CloseBackupableDB();
+  DestroyDB(dbname_, Options());
+
+  backupable_options_->destroy_old_data = false;
+  test_backup_env_->ClearWrittenFiles();
+  test_backup_env_->SetLimitDeleteFiles(0);
+  auto read_only_backup_engine =
+      BackupEngineReadOnly::NewReadOnlyBackupEngine(env_, *backupable_options_);
+  std::vector<BackupInfo> backup_info;
+  read_only_backup_engine->GetBackupInfo(&backup_info);
+  ASSERT_EQ(backup_info.size(), 2U);
+
+  RestoreOptions restore_options(false);
+  ASSERT_OK(read_only_backup_engine->RestoreDBFromLatestBackup(
+      dbname_, dbname_, restore_options));
+  delete read_only_backup_engine;
+  std::vector<std::string> should_have_written;
+  test_backup_env_->AssertWrittenFiles(should_have_written);
+
+  DB* db = OpenDB();
+  AssertExists(db, 0, 200);
+  delete db;
+}
+
+}  // anon namespace
 
 } //  namespace rocksdb