NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add Status check assertions for repair_test (#7455)

Browse Source 
Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/7455

Reviewed By: riversand963

Differential Revision: D23985283

Pulled By: ajkr

fbshipit-source-id: 5dd2be62350f6e31d13a1e7821cb848a37699c93
main
Andrew Kryczka 4 years ago committed by Facebook GitHub Bot
parent bd5d9e2a1d
commit 8115eb520d
8 changed files with 179 additions and 116 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      Makefile
  2. 19
      db/db_sst_test.cc
  3. 22
      db/db_test_util.cc
  4. 2
      db/db_test_util.h
  5. 71
      db/repair.cc
  6. 111
      db/repair_test.cc
  7. 2
      db/table_cache.cc
  8. 13
      db/wal_manager.cc

1
Makefile
Unescape View File

@ -609,6 +609,7 @@ ifdef ASSERT_STATUS_CHECKED
merger_test \ merger_test \
mock_env_test \ mock_env_test \
object_registry_test \ object_registry_test \
repair_test \
configurable_test \ configurable_test \
options_settable_test \ options_settable_test \
options_test \ options_test \

19
db/db_sst_test.cc
Unescape View File

@ -304,11 +304,14 @@ TEST_F(DBSSTTest, DBWithSstFileManager) {
dbfull()->TEST_WaitForFlushMemTable(); dbfull()->TEST_WaitForFlushMemTable();
dbfull()->TEST_WaitForCompact(); dbfull()->TEST_WaitForCompact();
// Verify that we are tracking all sst files in dbname_ // Verify that we are tracking all sst files in dbname_
ASSERT_EQ(sfm->GetTrackedFiles(), GetAllSSTFiles()); std::unordered_map<std::string, uint64_t> files_in_db;
ASSERT_OK(GetAllSSTFiles(&files_in_db));
ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db);
} }
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
auto files_in_db = GetAllSSTFiles(); std::unordered_map<std::string, uint64_t> files_in_db;
ASSERT_OK(GetAllSSTFiles(&files_in_db));
// Verify that we are tracking all sst files in dbname_ // Verify that we are tracking all sst files in dbname_
ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db); ASSERT_EQ(sfm->GetTrackedFiles(), files_in_db);
// Verify the total files size // Verify the total files size
@ -762,7 +765,8 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowed) {
ASSERT_OK(Flush()); ASSERT_OK(Flush());
uint64_t first_file_size = 0; uint64_t first_file_size = 0;
auto files_in_db = GetAllSSTFiles(&first_file_size); std::unordered_map<std::string, uint64_t> files_in_db;
ASSERT_OK(GetAllSSTFiles(&files_in_db, &first_file_size));
ASSERT_EQ(sfm->GetTotalSize(), first_file_size); ASSERT_EQ(sfm->GetTotalSize(), first_file_size);
// Set the maximum allowed space usage to the current total size // Set the maximum allowed space usage to the current total size
@ -802,7 +806,8 @@ TEST_F(DBSSTTest, CancellingCompactionsWorks) {
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
uint64_t total_file_size = 0; uint64_t total_file_size = 0;
auto files_in_db = GetAllSSTFiles(&total_file_size); std::unordered_map<std::string, uint64_t> files_in_db;
ASSERT_OK(GetAllSSTFiles(&files_in_db, &total_file_size));
// Set the maximum allowed space usage to the current total size // Set the maximum allowed space usage to the current total size
sfm->SetMaxAllowedSpaceUsage(2 * total_file_size + 1); sfm->SetMaxAllowedSpaceUsage(2 * total_file_size + 1);
@ -849,7 +854,8 @@ TEST_F(DBSSTTest, CancellingManualCompactionsWorks) {
} }
ASSERT_OK(Flush()); ASSERT_OK(Flush());
uint64_t total_file_size = 0; uint64_t total_file_size = 0;
auto files_in_db = GetAllSSTFiles(&total_file_size); std::unordered_map<std::string, uint64_t> files_in_db;
ASSERT_OK(GetAllSSTFiles(&files_in_db, &total_file_size));
// Set the maximum allowed space usage to the current total size // Set the maximum allowed space usage to the current total size
sfm->SetMaxAllowedSpaceUsage(2 * total_file_size + 1); sfm->SetMaxAllowedSpaceUsage(2 * total_file_size + 1);
@ -959,7 +965,8 @@ TEST_F(DBSSTTest, DBWithMaxSpaceAllowedRandomized) {
} }
ASSERT_TRUE(bg_error_set); ASSERT_TRUE(bg_error_set);
uint64_t total_sst_files_size = 0; uint64_t total_sst_files_size = 0;
GetAllSSTFiles(&total_sst_files_size); std::unordered_map<std::string, uint64_t> files_in_db;
ASSERT_OK(GetAllSSTFiles(&files_in_db, &total_sst_files_size));
ASSERT_GE(total_sst_files_size, limit_mb * 1024 * 1024); ASSERT_GE(total_sst_files_size, limit_mb * 1024 * 1024);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
} }

22
db/db_test_util.cc
Unescape View File

@ -1407,29 +1407,33 @@ void DBTestBase::CopyFile(const std::string& source,
ASSERT_OK(destfile->Close()); ASSERT_OK(destfile->Close());
} }
std::unordered_map<std::string, uint64_t> DBTestBase::GetAllSSTFiles( Status DBTestBase::GetAllSSTFiles(
uint64_t* total_size) { std::unordered_map<std::string, uint64_t>* sst_files,
std::unordered_map<std::string, uint64_t> res; uint64_t* total_size /* = nullptr */) {
if (total_size) { if (total_size) {
*total_size = 0; *total_size = 0;
} }
std::vector<std::string> files; std::vector<std::string> files;
env_->GetChildren(dbname_, &files); Status s = env_->GetChildren(dbname_, &files);
if (s.ok()) {
for (auto& file_name : files) { for (auto& file_name : files) {
uint64_t number; uint64_t number;
FileType type; FileType type;
std::string file_path = dbname_ + "/" + file_name;
if (ParseFileName(file_name, &number, &type) && type == kTableFile) { if (ParseFileName(file_name, &number, &type) && type == kTableFile) {
std::string file_path = dbname_ + "/" + file_name;
uint64_t file_size = 0; uint64_t file_size = 0;
env_->GetFileSize(file_path, &file_size); s = env_->GetFileSize(file_path, &file_size);
res[file_path] = file_size; if (!s.ok()) {
break;
}
(*sst_files)[file_path] = file_size;
if (total_size) { if (total_size) {
*total_size += file_size; *total_size += file_size;
} }
} }
} }
return res; }
return s;
} }
std::vector<std::uint64_t> DBTestBase::ListTableFiles(Env* env, std::vector<std::uint64_t> DBTestBase::ListTableFiles(Env* env,

2
db/db_test_util.h
Unescape View File

@ -1145,7 +1145,7 @@ class DBTestBase : public testing::Test {
void CopyFile(const std::string& source, const std::string& destination, void CopyFile(const std::string& source, const std::string& destination,
uint64_t size = 0); uint64_t size = 0);
std::unordered_map<std::string, uint64_t> GetAllSSTFiles( Status GetAllSSTFiles(std::unordered_map<std::string, uint64_t>* sst_files,
uint64_t* total_size = nullptr); uint64_t* total_size = nullptr);
std::vector<std::uint64_t> ListTableFiles(Env* env, const std::string& path); std::vector<std::uint64_t> ListTableFiles(Env* env, const std::string& path);

71
db/repair.cc
Unescape View File

@ -119,7 +119,8 @@ class Repairer {
raw_table_cache_.get(), &wb_, &wc_, raw_table_cache_.get(), &wb_, &wc_,
/*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr), /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr),
next_file_number_(1), next_file_number_(1),
db_lock_(nullptr) { db_lock_(nullptr),
closed_(false) {
for (const auto& cfd : column_families) { for (const auto& cfd : column_families) {
cf_name_to_opts_[cfd.name] = cfd.options; cf_name_to_opts_[cfd.name] = cfd.options;
} }
@ -163,12 +164,19 @@ class Repairer {
return status; return status;
} }
~Repairer() { Status Close() {
Status s = Status::OK();
if (!closed_) {
if (db_lock_ != nullptr) { if (db_lock_ != nullptr) {
env_->UnlockFile(db_lock_); s = env_->UnlockFile(db_lock_);
db_lock_ = nullptr;
} }
delete table_cache_; closed_ = true;
} }
return s;
}
~Repairer() { Close().PermitUncheckedError(); }
Status Run() { Status Run() {
Status status = env_->LockFile(LockFileName(dbname_), &db_lock_); Status status = env_->LockFile(LockFileName(dbname_), &db_lock_);
@ -176,18 +184,21 @@ class Repairer {
return status; return status;
} }
status = FindFiles(); status = FindFiles();
DBImpl* db_impl = nullptr;
if (status.ok()) { if (status.ok()) {
// Discard older manifests and start a fresh one // Discard older manifests and start a fresh one
for (size_t i = 0; i < manifests_.size(); i++) { for (size_t i = 0; i < manifests_.size(); i++) {
ArchiveFile(dbname_ + "/" + manifests_[i]); ArchiveFile(dbname_ + "/" + manifests_[i]);
} }
// Just create a DBImpl temporarily so we can reuse NewDB() // Just create a DBImpl temporarily so we can reuse NewDB()
DBImpl* db_impl = new DBImpl(db_options_, dbname_); db_impl = new DBImpl(db_options_, dbname_);
// Also use this temp DBImpl to get a session id // Also use this temp DBImpl to get a session id
db_impl->GetDbSessionId(db_session_id_); status = db_impl->GetDbSessionId(db_session_id_);
}
if (status.ok()) {
status = db_impl->NewDB(/*new_filenames=*/nullptr); status = db_impl->NewDB(/*new_filenames=*/nullptr);
delete db_impl;
} }
delete db_impl;
if (status.ok()) { if (status.ok()) {
// Recover using the fresh manifest created by NewDB() // Recover using the fresh manifest created by NewDB()
@ -242,7 +253,7 @@ class Repairer {
const ColumnFamilyOptions unknown_cf_opts_; const ColumnFamilyOptions unknown_cf_opts_;
const bool create_unknown_cfs_; const bool create_unknown_cfs_;
std::shared_ptr<Cache> raw_table_cache_; std::shared_ptr<Cache> raw_table_cache_;
TableCache* table_cache_; std::unique_ptr<TableCache> table_cache_;
WriteBufferManager wb_; WriteBufferManager wb_;
WriteController wc_; WriteController wc_;
VersionSet vset_; VersionSet vset_;
@ -257,6 +268,7 @@ class Repairer {
// Lock over the persistent DB state. Non-nullptr iff successfully // Lock over the persistent DB state. Non-nullptr iff successfully
// acquired. // acquired.
FileLock* db_lock_; FileLock* db_lock_;
bool closed_;
Status FindFiles() { Status FindFiles() {
std::vector<std::string> filenames; std::vector<std::string> filenames;
@ -385,15 +397,16 @@ class Repairer {
record.size(), Status::Corruption("log record too small")); record.size(), Status::Corruption("log record too small"));
continue; continue;
} }
WriteBatchInternal::SetContents(&batch, record); Status record_status = WriteBatchInternal::SetContents(&batch, record);
status = if (record_status.ok()) {
record_status =
WriteBatchInternal::InsertInto(&batch, cf_mems, nullptr, nullptr); WriteBatchInternal::InsertInto(&batch, cf_mems, nullptr, nullptr);
if (status.ok()) { }
if (record_status.ok()) {
counter += WriteBatchInternal::Count(&batch); counter += WriteBatchInternal::Count(&batch);
} else { } else {
ROCKS_LOG_WARN(db_options_.info_log, "Log #%" PRIu64 ": ignoring %s", ROCKS_LOG_WARN(db_options_.info_log, "Log #%" PRIu64 ": ignoring %s",
log, status.ToString().c_str()); log, record_status.ToString().c_str());
status = Status::OK(); // Keep going with rest of file
} }
} }
@ -430,7 +443,7 @@ class Repairer {
IOStatus io_s; IOStatus io_s;
status = BuildTable( status = BuildTable(
dbname_, /* versions */ nullptr, env_, &fs, *cfd->ioptions(), dbname_, /* versions */ nullptr, env_, &fs, *cfd->ioptions(),
*cfd->GetLatestMutableCFOptions(), env_options_, table_cache_, *cfd->GetLatestMutableCFOptions(), env_options_, table_cache_.get(),
iter.get(), std::move(range_del_iters), &meta, iter.get(), std::move(range_del_iters), &meta,
nullptr /* blob_file_additions */, cfd->internal_comparator(), nullptr /* blob_file_additions */, cfd->internal_comparator(),
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(), cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
@ -623,7 +636,7 @@ class Repairer {
new_dir.assign(fname.data(), slash - fname.data()); new_dir.assign(fname.data(), slash - fname.data());
} }
new_dir.append("/lost"); new_dir.append("/lost");
env_->CreateDir(new_dir); // Ignore error env_->CreateDir(new_dir).PermitUncheckedError(); // Ignore error
std::string new_file = new_dir; std::string new_file = new_dir;
new_file.append("/"); new_file.append("/");
new_file.append((slash == nullptr) ? fname.c_str() : slash + 1); new_file.append((slash == nullptr) ? fname.c_str() : slash + 1);
@ -655,12 +668,16 @@ Status RepairDB(const std::string& dbname, const DBOptions& db_options,
) { ) {
ColumnFamilyOptions default_cf_opts; ColumnFamilyOptions default_cf_opts;
Status status = GetDefaultCFOptions(column_families, &default_cf_opts); Status status = GetDefaultCFOptions(column_families, &default_cf_opts);
if (status.ok()) { if (!status.ok()) {
Repairer repairer(dbname, db_options, column_families, return status;
default_cf_opts, }
Repairer repairer(dbname, db_options, column_families, default_cf_opts,
ColumnFamilyOptions() /* unknown_cf_opts */, ColumnFamilyOptions() /* unknown_cf_opts */,
false /* create_unknown_cfs */); false /* create_unknown_cfs */);
status = repairer.Run(); status = repairer.Run();
if (status.ok()) {
status = repairer.Close();
} }
return status; return status;
} }
@ -670,25 +687,33 @@ Status RepairDB(const std::string& dbname, const DBOptions& db_options,
const ColumnFamilyOptions& unknown_cf_opts) { const ColumnFamilyOptions& unknown_cf_opts) {
ColumnFamilyOptions default_cf_opts; ColumnFamilyOptions default_cf_opts;
Status status = GetDefaultCFOptions(column_families, &default_cf_opts); Status status = GetDefaultCFOptions(column_families, &default_cf_opts);
if (status.ok()) { if (!status.ok()) {
Repairer repairer(dbname, db_options, return status;
column_families, default_cf_opts, }
Repairer repairer(dbname, db_options, column_families, default_cf_opts,
unknown_cf_opts, true /* create_unknown_cfs */); unknown_cf_opts, true /* create_unknown_cfs */);
status = repairer.Run(); status = repairer.Run();
if (status.ok()) {
status = repairer.Close();
} }
return status; return status;
} }
Status RepairDB(const std::string& dbname, const Options& options) { Status RepairDB(const std::string& dbname, const Options& options) {
Options opts(options); Options opts(options);
DBOptions db_options(opts); DBOptions db_options(opts);
ColumnFamilyOptions cf_options(opts); ColumnFamilyOptions cf_options(opts);
Repairer repairer(dbname, db_options, Repairer repairer(dbname, db_options,
{}, cf_options /* default_cf_opts */, {}, cf_options /* default_cf_opts */,
cf_options /* unknown_cf_opts */, cf_options /* unknown_cf_opts */,
true /* create_unknown_cfs */); true /* create_unknown_cfs */);
return repairer.Run(); Status status = repairer.Run();
if (status.ok()) {
status = repairer.Close();
}
return status;
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

111
db/repair_test.cc
Unescape View File

@ -24,10 +24,13 @@ class RepairTest : public DBTestBase {
public: public:
RepairTest() : DBTestBase("/repair_test", /*env_do_fsync=*/true) {} RepairTest() : DBTestBase("/repair_test", /*env_do_fsync=*/true) {}
std::string GetFirstSstPath() { Status GetFirstSstPath(std::string* first_sst_path) {
assert(first_sst_path != nullptr);
first_sst_path->clear();
uint64_t manifest_size; uint64_t manifest_size;
std::vector<std::string> files; std::vector<std::string> files;
db_->GetLiveFiles(files, &manifest_size); Status s = db_->GetLiveFiles(files, &manifest_size);
if (s.ok()) {
auto sst_iter = auto sst_iter =
std::find_if(files.begin(), files.end(), [](const std::string& file) { std::find_if(files.begin(), files.end(), [](const std::string& file) {
uint64_t number; uint64_t number;
@ -35,17 +38,19 @@ class RepairTest : public DBTestBase {
bool ok = ParseFileName(file, &number, &type); bool ok = ParseFileName(file, &number, &type);
return ok && type == kTableFile; return ok && type == kTableFile;
}); });
return sst_iter == files.end() ? "" : dbname_ + *sst_iter; *first_sst_path = sst_iter == files.end() ? "" : dbname_ + *sst_iter;
}
return s;
} }
}; };
TEST_F(RepairTest, LostManifest) { TEST_F(RepairTest, LostManifest) {
// Add a couple SST files, delete the manifest, and verify RepairDB() saves // Add a couple SST files, delete the manifest, and verify RepairDB() saves
// the day. // the day.
Put("key", "val"); ASSERT_OK(Put("key", "val"));
Flush(); ASSERT_OK(Flush());
Put("key2", "val2"); ASSERT_OK(Put("key2", "val2"));
Flush(); ASSERT_OK(Flush());
// Need to get path before Close() deletes db_, but delete it after Close() to // Need to get path before Close() deletes db_, but delete it after Close() to
// ensure Close() didn't change the manifest. // ensure Close() didn't change the manifest.
std::string manifest_path = std::string manifest_path =
@ -63,10 +68,10 @@ TEST_F(RepairTest, LostManifest) {
TEST_F(RepairTest, CorruptManifest) { TEST_F(RepairTest, CorruptManifest) {
// Manifest is in an invalid format. Expect a full recovery. // Manifest is in an invalid format. Expect a full recovery.
Put("key", "val"); ASSERT_OK(Put("key", "val"));
Flush(); ASSERT_OK(Flush());
Put("key2", "val2"); ASSERT_OK(Put("key2", "val2"));
Flush(); ASSERT_OK(Flush());
// Need to get path before Close() deletes db_, but overwrite it after Close() // Need to get path before Close() deletes db_, but overwrite it after Close()
// to ensure Close() didn't change the manifest. // to ensure Close() didn't change the manifest.
std::string manifest_path = std::string manifest_path =
@ -76,7 +81,7 @@ TEST_F(RepairTest, CorruptManifest) {
ASSERT_OK(env_->FileExists(manifest_path)); ASSERT_OK(env_->FileExists(manifest_path));
LegacyFileSystemWrapper fs(env_); LegacyFileSystemWrapper fs(env_);
CreateFile(&fs, manifest_path, "blah", false /* use_fsync */); ASSERT_OK(CreateFile(&fs, manifest_path, "blah", false /* use_fsync */));
ASSERT_OK(RepairDB(dbname_, CurrentOptions())); ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
Reopen(CurrentOptions()); Reopen(CurrentOptions());
@ -87,13 +92,13 @@ TEST_F(RepairTest, CorruptManifest) {
TEST_F(RepairTest, IncompleteManifest) { TEST_F(RepairTest, IncompleteManifest) {
// In this case, the manifest is valid but does not reference all of the SST // In this case, the manifest is valid but does not reference all of the SST
// files. Expect a full recovery. // files. Expect a full recovery.
Put("key", "val"); ASSERT_OK(Put("key", "val"));
Flush(); ASSERT_OK(Flush());
std::string orig_manifest_path = std::string orig_manifest_path =
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
CopyFile(orig_manifest_path, orig_manifest_path + ".tmp"); CopyFile(orig_manifest_path, orig_manifest_path + ".tmp");
Put("key2", "val2"); ASSERT_OK(Put("key2", "val2"));
Flush(); ASSERT_OK(Flush());
// Need to get path before Close() deletes db_, but overwrite it after Close() // Need to get path before Close() deletes db_, but overwrite it after Close()
// to ensure Close() didn't change the manifest. // to ensure Close() didn't change the manifest.
std::string new_manifest_path = std::string new_manifest_path =
@ -113,10 +118,10 @@ TEST_F(RepairTest, IncompleteManifest) {
TEST_F(RepairTest, PostRepairSstFileNumbering) { TEST_F(RepairTest, PostRepairSstFileNumbering) {
// Verify after a DB is repaired, new files will be assigned higher numbers // Verify after a DB is repaired, new files will be assigned higher numbers
// than old files. // than old files.
Put("key", "val"); ASSERT_OK(Put("key", "val"));
Flush(); ASSERT_OK(Flush());
Put("key2", "val2"); ASSERT_OK(Put("key2", "val2"));
Flush(); ASSERT_OK(Flush());
uint64_t pre_repair_file_num = dbfull()->TEST_Current_Next_FileNo(); uint64_t pre_repair_file_num = dbfull()->TEST_Current_Next_FileNo();
Close(); Close();
@ -130,11 +135,12 @@ TEST_F(RepairTest, PostRepairSstFileNumbering) {
TEST_F(RepairTest, LostSst) { TEST_F(RepairTest, LostSst) {
// Delete one of the SST files but preserve the manifest that refers to it, // Delete one of the SST files but preserve the manifest that refers to it,
// then verify the DB is still usable for the intact SST. // then verify the DB is still usable for the intact SST.
Put("key", "val"); ASSERT_OK(Put("key", "val"));
Flush(); ASSERT_OK(Flush());
Put("key2", "val2"); ASSERT_OK(Put("key2", "val2"));
Flush(); ASSERT_OK(Flush());
auto sst_path = GetFirstSstPath(); std::string sst_path;
ASSERT_OK(GetFirstSstPath(&sst_path));
ASSERT_FALSE(sst_path.empty()); ASSERT_FALSE(sst_path.empty());
ASSERT_OK(env_->DeleteFile(sst_path)); ASSERT_OK(env_->DeleteFile(sst_path));
@ -149,15 +155,16 @@ TEST_F(RepairTest, LostSst) {
TEST_F(RepairTest, CorruptSst) { TEST_F(RepairTest, CorruptSst) {
// Corrupt one of the SST files but preserve the manifest that refers to it, // Corrupt one of the SST files but preserve the manifest that refers to it,
// then verify the DB is still usable for the intact SST. // then verify the DB is still usable for the intact SST.
Put("key", "val"); ASSERT_OK(Put("key", "val"));
Flush(); ASSERT_OK(Flush());
Put("key2", "val2"); ASSERT_OK(Put("key2", "val2"));
Flush(); ASSERT_OK(Flush());
auto sst_path = GetFirstSstPath(); std::string sst_path;
ASSERT_OK(GetFirstSstPath(&sst_path));
ASSERT_FALSE(sst_path.empty()); ASSERT_FALSE(sst_path.empty());
LegacyFileSystemWrapper fs(env_); LegacyFileSystemWrapper fs(env_);
CreateFile(&fs, sst_path, "blah", false /* use_fsync */); ASSERT_OK(CreateFile(&fs, sst_path, "blah", false /* use_fsync */));
Close(); Close();
ASSERT_OK(RepairDB(dbname_, CurrentOptions())); ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
@ -170,13 +177,16 @@ TEST_F(RepairTest, CorruptSst) {
TEST_F(RepairTest, UnflushedSst) { TEST_F(RepairTest, UnflushedSst) {
// This test case invokes repair while some data is unflushed, then verifies // This test case invokes repair while some data is unflushed, then verifies
// that data is in the db. // that data is in the db.
Put("key", "val"); ASSERT_OK(Put("key", "val"));
VectorLogPtr wal_files; VectorLogPtr wal_files;
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
ASSERT_EQ(wal_files.size(), 1); ASSERT_EQ(wal_files.size(), 1);
{
uint64_t total_ssts_size; uint64_t total_ssts_size;
GetAllSSTFiles(&total_ssts_size); std::unordered_map<std::string, uint64_t> sst_files;
ASSERT_OK(GetAllSSTFiles(&sst_files, &total_ssts_size));
ASSERT_EQ(total_ssts_size, 0); ASSERT_EQ(total_ssts_size, 0);
}
// Need to get path before Close() deletes db_, but delete it after Close() to // Need to get path before Close() deletes db_, but delete it after Close() to
// ensure Close() didn't change the manifest. // ensure Close() didn't change the manifest.
std::string manifest_path = std::string manifest_path =
@ -190,8 +200,12 @@ TEST_F(RepairTest, UnflushedSst) {
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
ASSERT_EQ(wal_files.size(), 0); ASSERT_EQ(wal_files.size(), 0);
GetAllSSTFiles(&total_ssts_size); {
uint64_t total_ssts_size;
std::unordered_map<std::string, uint64_t> sst_files;
ASSERT_OK(GetAllSSTFiles(&sst_files, &total_ssts_size));
ASSERT_GT(total_ssts_size, 0); ASSERT_GT(total_ssts_size, 0);
}
ASSERT_EQ(Get("key"), "val"); ASSERT_EQ(Get("key"), "val");
} }
@ -199,14 +213,17 @@ TEST_F(RepairTest, SeparateWalDir) {
do { do {
Options options = CurrentOptions(); Options options = CurrentOptions();
DestroyAndReopen(options); DestroyAndReopen(options);
Put("key", "val"); ASSERT_OK(Put("key", "val"));
Put("foo", "bar"); ASSERT_OK(Put("foo", "bar"));
VectorLogPtr wal_files; VectorLogPtr wal_files;
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
ASSERT_EQ(wal_files.size(), 1); ASSERT_EQ(wal_files.size(), 1);
{
uint64_t total_ssts_size; uint64_t total_ssts_size;
GetAllSSTFiles(&total_ssts_size); std::unordered_map<std::string, uint64_t> sst_files;
ASSERT_OK(GetAllSSTFiles(&sst_files, &total_ssts_size));
ASSERT_EQ(total_ssts_size, 0); ASSERT_EQ(total_ssts_size, 0);
}
std::string manifest_path = std::string manifest_path =
DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo()); DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
@ -221,8 +238,12 @@ TEST_F(RepairTest, SeparateWalDir) {
Reopen(options); Reopen(options);
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files)); ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
ASSERT_EQ(wal_files.size(), 0); ASSERT_EQ(wal_files.size(), 0);
GetAllSSTFiles(&total_ssts_size); {
uint64_t total_ssts_size;
std::unordered_map<std::string, uint64_t> sst_files;
ASSERT_OK(GetAllSSTFiles(&sst_files, &total_ssts_size));
ASSERT_GT(total_ssts_size, 0); ASSERT_GT(total_ssts_size, 0);
}
ASSERT_EQ(Get("key"), "val"); ASSERT_EQ(Get("key"), "val");
ASSERT_EQ(Get("foo"), "bar"); ASSERT_EQ(Get("foo"), "bar");
@ -238,13 +259,13 @@ TEST_F(RepairTest, RepairMultipleColumnFamilies) {
CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions()); CreateAndReopenWithCF({"pikachu1", "pikachu2"}, CurrentOptions());
for (int i = 0; i < kNumCfs; ++i) { for (int i = 0; i < kNumCfs; ++i) {
for (int j = 0; j < kEntriesPerCf; ++j) { for (int j = 0; j < kEntriesPerCf; ++j) {
Put(i, "key" + ToString(j), "val" + ToString(j)); ASSERT_OK(Put(i, "key" + ToString(j), "val" + ToString(j)));
if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) { if (j == kEntriesPerCf - 1 && i == kNumCfs - 1) {
// Leave one unflushed so we can verify WAL entries are properly // Leave one unflushed so we can verify WAL entries are properly
// associated with column families. // associated with column families.
continue; continue;
} }
Flush(i); ASSERT_OK(Flush(i));
} }
} }
@ -283,12 +304,12 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
std::vector<Options>{opts, rev_opts}); std::vector<Options>{opts, rev_opts});
for (int i = 0; i < kNumCfs; ++i) { for (int i = 0; i < kNumCfs; ++i) {
for (int j = 0; j < kEntriesPerCf; ++j) { for (int j = 0; j < kEntriesPerCf; ++j) {
Put(i, "key" + ToString(j), "val" + ToString(j)); ASSERT_OK(Put(i, "key" + ToString(j), "val" + ToString(j)));
if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) { if (i == kNumCfs - 1 && j == kEntriesPerCf - 1) {
// Leave one unflushed so we can verify RepairDB's flush logic // Leave one unflushed so we can verify RepairDB's flush logic
continue; continue;
} }
Flush(i); ASSERT_OK(Flush(i));
} }
} }
Close(); Close();
@ -308,7 +329,7 @@ TEST_F(RepairTest, RepairColumnFamilyOptions) {
// Examine table properties to verify RepairDB() used the right options when // Examine table properties to verify RepairDB() used the right options when
// converting WAL->SST // converting WAL->SST
TablePropertiesCollection fname_to_props; TablePropertiesCollection fname_to_props;
db_->GetPropertiesOfAllTables(handles_[1], &fname_to_props); ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[1], &fname_to_props));
ASSERT_EQ(fname_to_props.size(), 2U); ASSERT_EQ(fname_to_props.size(), 2U);
for (const auto& fname_and_props : fname_to_props) { for (const auto& fname_and_props : fname_to_props) {
std::string comparator_name ( std::string comparator_name (
@ -342,8 +363,8 @@ TEST_F(RepairTest, DbNameContainsTrailingSlash) {
} }
} }
Put("key", "val"); ASSERT_OK(Put("key", "val"));
Flush(); ASSERT_OK(Flush());
Close(); Close();
ASSERT_OK(RepairDB(dbname_ + "/", CurrentOptions())); ASSERT_OK(RepairDB(dbname_ + "/", CurrentOptions()));

2
db/table_cache.cc
Unescape View File

@ -169,7 +169,7 @@ Status TableCache::FindTable(const ReadOptions& ro,
const_cast<bool*>(&no_io)); const_cast<bool*>(&no_io));
if (*handle == nullptr) { if (*handle == nullptr) {
if (no_io) { // Don't do IO and return a not-found status if (no_io) {
return Status::Incomplete("Table not found in table_cache, no_io is set"); return Status::Incomplete("Table not found in table_cache, no_io is set");
} }
MutexLock load_lock(loader_mutex_.get(key)); MutexLock load_lock(loader_mutex_.get(key));

13
db/wal_manager.cc
Unescape View File

@ -492,14 +492,19 @@ Status WalManager::ReadFirstLine(const std::string& fname,
// TODO read record's till the first no corrupt entry? // TODO read record's till the first no corrupt entry?
} else { } else {
WriteBatch batch; WriteBatch batch;
WriteBatchInternal::SetContents(&batch, record); // We can overwrite an existing non-OK Status since it'd only reach here
// with `paranoid_checks == false`.
status = WriteBatchInternal::SetContents(&batch, record);
if (status.ok()) {
*sequence = WriteBatchInternal::Sequence(&batch); *sequence = WriteBatchInternal::Sequence(&batch);
return Status::OK(); return status;
}
} }
} }
// ReadRecord returns false on EOF, which means that the log file is empty. we // ReadRecord might have returned false on EOF, which means that the log file
// return status.ok() in that case and set sequence number to 0 // is empty. Or, a failure may have occurred while processing the first entry.
// In any case, return status and set sequence number to 0.
*sequence = 0; *sequence = 0;
return status; return status;
} }

Write Preview
Loading…
Cancel
Save