From 6bbfa1874bf82c01b1b7106f732ab9c721c2fdb8 Mon Sep 17 00:00:00 2001 From: sdong Date: Mon, 23 Nov 2015 16:05:09 -0800 Subject: [PATCH] BackupDB to have a mode to use file size in file name Summary: Getting file size from all the backup files can take a long time. In some cases, the sizes are available in file names. We allow a mode to get those sizes from file name. Test Plan: Make some unit tests in backupable_db_test to run in such a mode. Make sure RocksDB Lite builds too. Reviewers: IslamAbdelRahman, rven, yhchiang, kradhakrishnan, anthony, igor Reviewed By: igor Subscribers: muthu, asameet, leveldb, dhruba Differential Revision: https://reviews.facebook.net/D51243 --- include/rocksdb/utilities/backupable_db.h | 9 + utilities/backupable/backupable_db.cc | 68 +++- utilities/backupable/backupable_db_test.cc | 360 ++++++++++-------- utilities/backupable/backupable_db_testutil.h | 15 + 4 files changed, 295 insertions(+), 157 deletions(-) create mode 100644 utilities/backupable/backupable_db_testutil.h diff --git a/include/rocksdb/utilities/backupable_db.h b/include/rocksdb/utilities/backupable_db.h index 5c3275084..f5f394c22 100644 --- a/include/rocksdb/utilities/backupable_db.h +++ b/include/rocksdb/utilities/backupable_db.h @@ -88,6 +88,14 @@ struct BackupableDBOptions { // *turn it on only if you know what you're doing* bool share_files_with_checksum; + // Try to use the file size in file name instead of getting size from HDFS, + // if the file is generated with options.share_files_with_checksum = true. + // This is a temporary solution to reduce the backupable Db open latency when + // There are too many sst files. Will remove the option after we have a + // permanent solution. + // Default: false + bool use_file_size_in_file_name; + // Up to this many background threads will copy files for CreateNewBackup() // and RestoreDBFromBackup() // Default: 1 @@ -117,6 +125,7 @@ struct BackupableDBOptions { backup_rate_limit(_backup_rate_limit), restore_rate_limit(_restore_rate_limit), share_files_with_checksum(false), + use_file_size_in_file_name(false), max_background_operations(_max_background_operations), callback_trigger_interval_size(_callback_trigger_interval_size) { assert(share_table_files || !share_files_with_checksum); diff --git a/utilities/backupable/backupable_db.cc b/utilities/backupable/backupable_db.cc index 16f6d527b..4bd597cf4 100644 --- a/utilities/backupable/backupable_db.cc +++ b/utilities/backupable/backupable_db.cc @@ -180,7 +180,8 @@ class BackupEngineImpl : public BackupEngine { return files_; } - Status LoadFromFile(const std::string& backup_dir); + Status LoadFromFile(const std::string& backup_dir, + bool use_size_in_file_name); Status StoreToFile(bool sync); std::string GetInfoString() { @@ -542,7 +543,8 @@ Status BackupEngineImpl::Initialize() { } else { // Load data from storage // load the backups if any for (auto& backup : backups_) { - Status s = backup.second->LoadFromFile(options_.backup_dir); + Status s = backup.second->LoadFromFile( + options_.backup_dir, options_.use_file_size_in_file_name); if (!s.ok()) { Log(options_.info_log, "Backup %u corrupted -- %s", backup.first, s.ToString().c_str()); @@ -1451,6 +1453,55 @@ Status BackupEngineImpl::BackupMeta::Delete(bool delete_meta) { return s; } +namespace { +bool ParseStrToUint64(const std::string& str, uint64_t* out) { + try { + unsigned long ul = std::stoul(str); + *out = static_cast(ul); + return true; + } catch (const std::invalid_argument& e) { + return false; + } catch (const std::out_of_range& e) { + return false; + } +} + +// Parse file name in the format of +// "shared_checksum/__.sst, and fill `size` with +// the parsed part. +// Will also accept only name part, or a file path in URL format. +// if file name doesn't have the extension of "sst", or doesn't have '_' as a +// part of the file name, or we can't parse a number from the sub string +// between the last '_' and '.', return false. +bool GetFileSizeFromBackupFileName(const std::string full_name, + uint64_t* size) { + auto dot_pos = full_name.find_last_of('.'); + if (dot_pos == std::string::npos) { + return false; + } + if (full_name.substr(dot_pos + 1) != "sst") { + return false; + } + auto last_underscore_pos = full_name.find_last_of('_'); + if (last_underscore_pos == std::string::npos) { + return false; + } + if (dot_pos <= last_underscore_pos + 2) { + return false; + } + return ParseStrToUint64(full_name.substr(last_underscore_pos + 1, + dot_pos - last_underscore_pos - 1), + size); +} +} // namespace + +namespace test { +bool TEST_GetFileSizeFromBackupFileName(const std::string full_name, + uint64_t* size) { + return GetFileSizeFromBackupFileName(full_name, size); +} +} // namespace test + // each backup meta file is of the format: // // @@ -1458,8 +1509,8 @@ Status BackupEngineImpl::BackupMeta::Delete(bool delete_meta) { // // // ... -Status BackupEngineImpl::BackupMeta::LoadFromFile( - const std::string& backup_dir) { +Status BackupEngineImpl::BackupMeta::LoadFromFile(const std::string& backup_dir, + bool use_size_in_file_name) { assert(Empty()); Status s; unique_ptr backup_meta_file; @@ -1501,9 +1552,12 @@ Status BackupEngineImpl::BackupMeta::LoadFromFile( if (file_info) { size = file_info->size; } else { - s = env_->GetFileSize(backup_dir + "/" + filename, &size); - if (!s.ok()) { - return s; + if (!use_size_in_file_name || + !GetFileSizeFromBackupFileName(filename, &size)) { + s = env_->GetFileSize(backup_dir + "/" + filename, &size); + if (!s.ok()) { + return s; + } } } diff --git a/utilities/backupable/backupable_db_test.cc b/utilities/backupable/backupable_db_test.cc index 13917bd4a..b4f56bf7a 100644 --- a/utilities/backupable/backupable_db_test.cc +++ b/utilities/backupable/backupable_db_test.cc @@ -26,6 +26,7 @@ #include "util/testutil.h" #include "util/auto_roll_logger.h" #include "util/mock_env.h" +#include "utilities/backupable/backupable_db_testutil.h" namespace rocksdb { @@ -448,9 +449,15 @@ class BackupableDBTest : public testing::Test { return db; } + void OpenDBAndBackupEngineShareWithChecksum( + bool destroy_old_data = false, bool dummy = false, + bool share_table_files = true, bool share_with_checksums = false) { + backupable_options_->share_files_with_checksum = share_with_checksums; + OpenDBAndBackupEngine(destroy_old_data, dummy, share_with_checksums); + } + void OpenDBAndBackupEngine(bool destroy_old_data = false, bool dummy = false, - bool share_table_files = true, - bool share_with_checksums = false) { + bool share_table_files = true) { // reset all the defaults test_backup_env_->SetLimitWrittenFiles(1000000); test_db_env_->SetLimitWrittenFiles(1000000); @@ -466,7 +473,6 @@ class BackupableDBTest : public testing::Test { db_.reset(db); backupable_options_->destroy_old_data = destroy_old_data; backupable_options_->share_table_files = share_table_files; - backupable_options_->share_files_with_checksum = share_with_checksums; BackupEngine* backup_engine; ASSERT_OK(BackupEngine::Open(test_db_env_.get(), *backupable_options_, &backup_engine)); @@ -552,8 +558,10 @@ class BackupableDBTest : public testing::Test { // options Options options_; - unique_ptr backupable_options_; std::shared_ptr logger_; + + protected: + unique_ptr backupable_options_; }; // BackupableDBTest void AppendPath(const std::string& path, std::vector& v) { @@ -562,6 +570,201 @@ void AppendPath(const std::string& path, std::vector& v) { } } +class BackupableDBTestWithParam : public BackupableDBTest, + public testing::WithParamInterface { + public: + BackupableDBTestWithParam() { + backupable_options_->share_files_with_checksum = + backupable_options_->use_file_size_in_file_name = GetParam(); + } +}; + +// This test verifies that the verifyBackup method correctly identifies +// invalid backups +TEST_P(BackupableDBTestWithParam, VerifyBackup) { + const int keys_iteration = 5000; + Random rnd(6); + Status s; + OpenDBAndBackupEngine(true); + // create five backups + for (int i = 0; i < 5; ++i) { + FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1)); + ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true)); + } + CloseDBAndBackupEngine(); + + OpenDBAndBackupEngine(); + // ---------- case 1. - valid backup ----------- + ASSERT_TRUE(backup_engine_->VerifyBackup(1).ok()); + + // ---------- case 2. - delete a file -----------i + file_manager_->DeleteRandomFileInDir(backupdir_ + "/private/1"); + ASSERT_TRUE(backup_engine_->VerifyBackup(1).IsNotFound()); + + // ---------- case 3. - corrupt a file ----------- + std::string append_data = "Corrupting a random file"; + file_manager_->AppendToRandomFileInDir(backupdir_ + "/private/2", + append_data); + ASSERT_TRUE(backup_engine_->VerifyBackup(2).IsCorruption()); + + // ---------- case 4. - invalid backup ----------- + ASSERT_TRUE(backup_engine_->VerifyBackup(6).IsNotFound()); + CloseDBAndBackupEngine(); +} + +// open DB, write, close DB, backup, restore, repeat +TEST_P(BackupableDBTestWithParam, OfflineIntegrationTest) { + // has to be a big number, so that it triggers the memtable flush + const int keys_iteration = 5000; + const int max_key = keys_iteration * 4 + 10; + // first iter -- flush before backup + // second iter -- don't flush before backup + for (int iter = 0; iter < 2; ++iter) { + // delete old data + DestroyDB(dbname_, Options()); + bool destroy_data = true; + + // every iteration -- + // 1. insert new data in the DB + // 2. backup the DB + // 3. destroy the db + // 4. restore the db, check everything is still there + for (int i = 0; i < 5; ++i) { + // in last iteration, put smaller amount of data, + int fill_up_to = std::min(keys_iteration * (i + 1), max_key); + // ---- insert new data and back up ---- + OpenDBAndBackupEngine(destroy_data); + destroy_data = false; + FillDB(db_.get(), keys_iteration * i, fill_up_to); + ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0)); + CloseDBAndBackupEngine(); + DestroyDB(dbname_, Options()); + + // ---- make sure it's empty ---- + DB* db = OpenDB(); + AssertEmpty(db, 0, fill_up_to); + delete db; + + // ---- restore the DB ---- + OpenBackupEngine(); + if (i >= 3) { // test purge old backups + // when i == 4, purge to only 1 backup + // when i == 3, purge to 2 backups + ASSERT_OK(backup_engine_->PurgeOldBackups(5 - i)); + } + // ---- make sure the data is there --- + AssertBackupConsistency(0, 0, fill_up_to, max_key); + CloseBackupEngine(); + } + } +} + +// open DB, write, backup, write, backup, close, restore +TEST_P(BackupableDBTestWithParam, OnlineIntegrationTest) { + // has to be a big number, so that it triggers the memtable flush + const int keys_iteration = 5000; + const int max_key = keys_iteration * 4 + 10; + Random rnd(7); + // delete old data + DestroyDB(dbname_, Options()); + + OpenDBAndBackupEngine(true); + // write some data, backup, repeat + for (int i = 0; i < 5; ++i) { + if (i == 4) { + // delete backup number 2, online delete! + ASSERT_OK(backup_engine_->DeleteBackup(2)); + } + // in last iteration, put smaller amount of data, + // so that backups can share sst files + int fill_up_to = std::min(keys_iteration * (i + 1), max_key); + FillDB(db_.get(), keys_iteration * i, fill_up_to); + // we should get consistent results with flush_before_backup + // set to both true and false + ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2))); + } + // close and destroy + CloseDBAndBackupEngine(); + DestroyDB(dbname_, Options()); + + // ---- make sure it's empty ---- + DB* db = OpenDB(); + AssertEmpty(db, 0, max_key); + delete db; + + // ---- restore every backup and verify all the data is there ---- + OpenBackupEngine(); + for (int i = 1; i <= 5; ++i) { + if (i == 2) { + // we deleted backup 2 + Status s = backup_engine_->RestoreDBFromBackup(2, dbname_, dbname_); + ASSERT_TRUE(!s.ok()); + } else { + int fill_up_to = std::min(keys_iteration * i, max_key); + AssertBackupConsistency(i, 0, fill_up_to, max_key); + } + } + + // delete some backups -- this should leave only backups 3 and 5 alive + ASSERT_OK(backup_engine_->DeleteBackup(4)); + ASSERT_OK(backup_engine_->PurgeOldBackups(2)); + + std::vector backup_info; + backup_engine_->GetBackupInfo(&backup_info); + ASSERT_EQ(2UL, backup_info.size()); + + // check backup 3 + AssertBackupConsistency(3, 0, 3 * keys_iteration, max_key); + // check backup 5 + AssertBackupConsistency(5, 0, max_key); + + CloseBackupEngine(); +} + +INSTANTIATE_TEST_CASE_P(BackupableDBTestWithParam, BackupableDBTestWithParam, + ::testing::Bool()); + +TEST_F(BackupableDBTest, GetFileSizeFromBackupFileName) { + uint64_t size = 0; + + ASSERT_TRUE(test::TEST_GetFileSizeFromBackupFileName( + "shared_checksum/6580354_1874793674_65806675.sst", &size)); + ASSERT_EQ(65806675u, size); + + ASSERT_TRUE(test::TEST_GetFileSizeFromBackupFileName( + "hdfs://a.b:80/a/b/shared_checksum/6580354_1874793674_85806675.sst", + &size)); + ASSERT_EQ(85806675u, size); + + ASSERT_TRUE(test::TEST_GetFileSizeFromBackupFileName( + "6580354_1874793674_65806665.sst", &size)); + ASSERT_EQ(65806665u, size); + + ASSERT_TRUE(test::TEST_GetFileSizeFromBackupFileName( + "private/66/6580354_1874793674_65806666.sst", &size)); + ASSERT_EQ(65806666u, size); + + ASSERT_TRUE(!test::TEST_GetFileSizeFromBackupFileName( + "shared_checksum/6580354.sst", &size)); + + ASSERT_TRUE(!test::TEST_GetFileSizeFromBackupFileName( + "private/368/6592388.log", &size)); + + ASSERT_TRUE(!test::TEST_GetFileSizeFromBackupFileName( + "private/68/MANIFEST-6586581", &size)); + + ASSERT_TRUE( + !test::TEST_GetFileSizeFromBackupFileName("private/68/CURRENT", &size)); + + ASSERT_TRUE(!test::TEST_GetFileSizeFromBackupFileName( + "shared_checksum/6580354_1874793674_65806675.log", &size)); + + ASSERT_TRUE(!test::TEST_GetFileSizeFromBackupFileName( + "shared_checksum/6580354_1874793674_65806675", &size)); + + ASSERT_TRUE(!test::TEST_GetFileSizeFromBackupFileName("meta/368", &size)); +} + // this will make sure that backup does not copy the same file twice TEST_F(BackupableDBTest, NoDoubleCopy) { OpenDBAndBackupEngine(true, true); @@ -788,39 +991,6 @@ TEST_F(BackupableDBTest, CorruptionsTest) { AssertBackupConsistency(2, 0, keys_iteration * 2, keys_iteration * 5); } -// This test verifies that the verifyBackup method correctly identifies -// invalid backups -TEST_F(BackupableDBTest, VerifyBackup) { - const int keys_iteration = 5000; - Random rnd(6); - Status s; - OpenDBAndBackupEngine(true); - // create five backups - for (int i = 0; i < 5; ++i) { - FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1)); - ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true)); - } - CloseDBAndBackupEngine(); - - OpenDBAndBackupEngine(); - // ---------- case 1. - valid backup ----------- - ASSERT_TRUE(backup_engine_->VerifyBackup(1).ok()); - - // ---------- case 2. - delete a file -----------i - file_manager_->DeleteRandomFileInDir(backupdir_ + "/private/1"); - ASSERT_TRUE(backup_engine_->VerifyBackup(1).IsNotFound()); - - // ---------- case 3. - corrupt a file ----------- - std::string append_data = "Corrupting a random file"; - file_manager_->AppendToRandomFileInDir(backupdir_ + "/private/2", - append_data); - ASSERT_TRUE(backup_engine_->VerifyBackup(2).IsCorruption()); - - // ---------- case 4. - invalid backup ----------- - ASSERT_TRUE(backup_engine_->VerifyBackup(6).IsNotFound()); - CloseDBAndBackupEngine(); -} - // This test verifies we don't delete the latest backup when read-only option is // set TEST_F(BackupableDBTest, NoDeleteWithReadOnly) { @@ -855,115 +1025,6 @@ TEST_F(BackupableDBTest, NoDeleteWithReadOnly) { delete read_only_backup_engine; } -// open DB, write, close DB, backup, restore, repeat -TEST_F(BackupableDBTest, OfflineIntegrationTest) { - // has to be a big number, so that it triggers the memtable flush - const int keys_iteration = 5000; - const int max_key = keys_iteration * 4 + 10; - // first iter -- flush before backup - // second iter -- don't flush before backup - for (int iter = 0; iter < 2; ++iter) { - // delete old data - DestroyDB(dbname_, Options()); - bool destroy_data = true; - - // every iteration -- - // 1. insert new data in the DB - // 2. backup the DB - // 3. destroy the db - // 4. restore the db, check everything is still there - for (int i = 0; i < 5; ++i) { - // in last iteration, put smaller amount of data, - int fill_up_to = std::min(keys_iteration * (i + 1), max_key); - // ---- insert new data and back up ---- - OpenDBAndBackupEngine(destroy_data); - destroy_data = false; - FillDB(db_.get(), keys_iteration * i, fill_up_to); - ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), iter == 0)); - CloseDBAndBackupEngine(); - DestroyDB(dbname_, Options()); - - // ---- make sure it's empty ---- - DB* db = OpenDB(); - AssertEmpty(db, 0, fill_up_to); - delete db; - - // ---- restore the DB ---- - OpenBackupEngine(); - if (i >= 3) { // test purge old backups - // when i == 4, purge to only 1 backup - // when i == 3, purge to 2 backups - ASSERT_OK(backup_engine_->PurgeOldBackups(5 - i)); - } - // ---- make sure the data is there --- - AssertBackupConsistency(0, 0, fill_up_to, max_key); - CloseBackupEngine(); - } - } -} - -// open DB, write, backup, write, backup, close, restore -TEST_F(BackupableDBTest, OnlineIntegrationTest) { - // has to be a big number, so that it triggers the memtable flush - const int keys_iteration = 5000; - const int max_key = keys_iteration * 4 + 10; - Random rnd(7); - // delete old data - DestroyDB(dbname_, Options()); - - OpenDBAndBackupEngine(true); - // write some data, backup, repeat - for (int i = 0; i < 5; ++i) { - if (i == 4) { - // delete backup number 2, online delete! - ASSERT_OK(backup_engine_->DeleteBackup(2)); - } - // in last iteration, put smaller amount of data, - // so that backups can share sst files - int fill_up_to = std::min(keys_iteration * (i + 1), max_key); - FillDB(db_.get(), keys_iteration * i, fill_up_to); - // we should get consistent results with flush_before_backup - // set to both true and false - ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(rnd.Next() % 2))); - } - // close and destroy - CloseDBAndBackupEngine(); - DestroyDB(dbname_, Options()); - - // ---- make sure it's empty ---- - DB* db = OpenDB(); - AssertEmpty(db, 0, max_key); - delete db; - - // ---- restore every backup and verify all the data is there ---- - OpenBackupEngine(); - for (int i = 1; i <= 5; ++i) { - if (i == 2) { - // we deleted backup 2 - Status s = backup_engine_->RestoreDBFromBackup(2, dbname_, dbname_); - ASSERT_TRUE(!s.ok()); - } else { - int fill_up_to = std::min(keys_iteration * i, max_key); - AssertBackupConsistency(i, 0, fill_up_to, max_key); - } - } - - // delete some backups -- this should leave only backups 3 and 5 alive - ASSERT_OK(backup_engine_->DeleteBackup(4)); - ASSERT_OK(backup_engine_->PurgeOldBackups(2)); - - std::vector backup_info; - backup_engine_->GetBackupInfo(&backup_info); - ASSERT_EQ(2UL, backup_info.size()); - - // check backup 3 - AssertBackupConsistency(3, 0, 3 * keys_iteration, max_key); - // check backup 5 - AssertBackupConsistency(5, 0, max_key); - - CloseBackupEngine(); -} - TEST_F(BackupableDBTest, FailOverwritingBackups) { options_.write_buffer_size = 1024 * 1024 * 1024; // 1GB options_.disable_auto_compactions = true; @@ -1019,7 +1080,7 @@ TEST_F(BackupableDBTest, NoShareTableFiles) { // Verify that you can backup and restore with share_files_with_checksum on TEST_F(BackupableDBTest, ShareTableFilesWithChecksums) { const int keys_iteration = 5000; - OpenDBAndBackupEngine(true, false, true, true); + OpenDBAndBackupEngineShareWithChecksum(true, false, true, true); for (int i = 0; i < 5; ++i) { FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1)); ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), !!(i % 2))); @@ -1037,7 +1098,7 @@ TEST_F(BackupableDBTest, ShareTableFilesWithChecksums) { TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsTransition) { const int keys_iteration = 5000; // set share_files_with_checksum to false - OpenDBAndBackupEngine(true, false, true, false); + OpenDBAndBackupEngineShareWithChecksum(true, false, true, false); for (int i = 0; i < 5; ++i) { FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1)); ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true)); @@ -1050,7 +1111,7 @@ TEST_F(BackupableDBTest, ShareTableFilesWithChecksumsTransition) { } // set share_files_with_checksum to true and do some more backups - OpenDBAndBackupEngine(true, false, true, true); + OpenDBAndBackupEngineShareWithChecksum(true, false, true, true); for (int i = 5; i < 10; ++i) { FillDB(db_.get(), keys_iteration * i, keys_iteration * (i + 1)); ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), true)); @@ -1252,7 +1313,6 @@ TEST_F(BackupableDBTest, EnvFailures) { delete backup_engine; } } - } // anon namespace } // namespace rocksdb diff --git a/utilities/backupable/backupable_db_testutil.h b/utilities/backupable/backupable_db_testutil.h new file mode 100644 index 000000000..6c45f33ed --- /dev/null +++ b/utilities/backupable/backupable_db_testutil.h @@ -0,0 +1,15 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +#pragma once +#ifndef ROCKSDB_LITE +#include + +namespace rocksdb { +namespace test { +extern bool TEST_GetFileSizeFromBackupFileName(const std::string full_name, + uint64_t* size); +} // namespace test +} // namespace rocksdb +#endif // ROCKSDB_LITE