Override check consistency for DBImplSecondary (#5469)

Summary:
`DBImplSecondary` calls `CheckConsistency()` during open. In the past, `DBImplSecondary` did not override this function thus `DBImpl::CheckConsistency()` is called.
The following can happen. The secondary instance is performing consistency check which calls `GetFileSize(file_path)` but the file at `file_path` is deleted by the primary instance. `DBImpl::CheckConsistency` does not account for this and fails the consistency check. This is undesirable. The solution is that, we call `DBImpl::CheckConsistency()` first. If it passes, then we are good. If not, we give it a second chance and handles the case of file(s) being deleted.

Test plan (on dev server):
```
$make clean && make -j20 all
$./db_secondary_test
```
All other existing unit tests must pass as well.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5469

Differential Revision: D15861845

Pulled By: riversand963

fbshipit-source-id: 507d72392508caed3cd003bb2e2aa43f993dd597
main
Yanqin Jin 6 years ago committed by Facebook Github Bot
parent 671d15cbdd
commit 7d8d56413d
  1. 2
      db/db_impl/db_impl.cc
  2. 38
      db/db_impl/db_impl_secondary.cc
  3. 6
      db/db_impl/db_impl_secondary.h
  4. 40
      db/db_impl/db_secondary_test.cc

@ -3017,6 +3017,7 @@ Status DBImpl::CheckConsistency() {
mutex_.AssertHeld();
std::vector<LiveFileMetaData> metadata;
versions_->GetLiveFilesMetaData(&metadata);
TEST_SYNC_POINT("DBImpl::CheckConsistency:AfterGetLiveFilesMetaData");
std::string corruption_messages;
for (const auto& md : metadata) {
@ -3024,6 +3025,7 @@ Status DBImpl::CheckConsistency() {
std::string file_path = md.db_path + md.name;
uint64_t fsize = 0;
TEST_SYNC_POINT("DBImpl::CheckConsistency:BeforeGetFileSize");
Status s = env_->GetFileSize(file_path, &fsize);
if (!s.ok() &&
env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok()) {

@ -451,6 +451,44 @@ Status DBImplSecondary::NewIterators(
return Status::OK();
}
Status DBImplSecondary::CheckConsistency() {
mutex_.AssertHeld();
Status s = DBImpl::CheckConsistency();
// If DBImpl::CheckConsistency() which is stricter returns success, then we
// do not need to give a second chance.
if (s.ok()) {
return s;
}
// It's possible that DBImpl::CheckConssitency() can fail because the primary
// may have removed certain files, causing the GetFileSize(name) call to
// fail and returning a PathNotFound. In this case, we take a best-effort
// approach and just proceed.
TEST_SYNC_POINT_CALLBACK(
"DBImplSecondary::CheckConsistency:AfterFirstAttempt", &s);
std::vector<LiveFileMetaData> metadata;
versions_->GetLiveFilesMetaData(&metadata);
std::string corruption_messages;
for (const auto& md : metadata) {
// md.name has a leading "/".
std::string file_path = md.db_path + md.name;
uint64_t fsize = 0;
s = env_->GetFileSize(file_path, &fsize);
if (!s.ok() &&
(env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok() ||
s.IsPathNotFound())) {
s = Status::OK();
}
if (!s.ok()) {
corruption_messages +=
"Can't access " + md.name + ": " + s.ToString() + "\n";
}
}
return corruption_messages.empty() ? Status::OK()
: Status::Corruption(corruption_messages);
}
Status DBImplSecondary::TryCatchUpWithPrimary() {
assert(versions_.get() != nullptr);
assert(manifest_reader_.get() != nullptr);

@ -197,6 +197,12 @@ class DBImplSecondary : public DBImpl {
Status MaybeInitLogReader(uint64_t log_number,
log::FragmentBufferedReader** log_reader);
// Check if all live files exist on file system and that their file sizes
// matche to the in-memory records. It is possible that some live files may
// have been deleted by the primary. In this case, CheckConsistency() does
// not flag the missing file as inconsistency.
Status CheckConsistency() override;
protected:
// ColumnFamilyCollector is a write batch handler which does nothing
// except recording unique column family IDs

@ -705,6 +705,46 @@ TEST_F(DBSecondaryTest, CatchUpAfterFlush) {
iter3->Seek("key1");
ASSERT_FALSE(iter3->Valid());
}
TEST_F(DBSecondaryTest, CheckConsistencyWhenOpen) {
bool called = false;
Options options;
options.env = env_;
options.disable_auto_compactions = true;
Reopen(options);
SyncPoint::GetInstance()->DisableProcessing();
SyncPoint::GetInstance()->ClearAllCallBacks();
SyncPoint::GetInstance()->SetCallBack(
"DBImplSecondary::CheckConsistency:AfterFirstAttempt", [&](void* arg) {
ASSERT_NE(nullptr, arg);
called = true;
auto* s = reinterpret_cast<Status*>(arg);
ASSERT_NOK(*s);
});
SyncPoint::GetInstance()->LoadDependency(
{{"DBImpl::CheckConsistency:AfterGetLiveFilesMetaData",
"BackgroundCallCompaction:0"},
{"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles",
"DBImpl::CheckConsistency:BeforeGetFileSize"}});
SyncPoint::GetInstance()->EnableProcessing();
ASSERT_OK(Put("a", "value0"));
ASSERT_OK(Put("c", "value0"));
ASSERT_OK(Flush());
ASSERT_OK(Put("b", "value1"));
ASSERT_OK(Put("d", "value1"));
ASSERT_OK(Flush());
port::Thread thread([this]() {
Options opts;
opts.env = env_;
opts.max_open_files = -1;
OpenSecondary(opts);
});
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
ASSERT_OK(dbfull()->TEST_WaitForCompact());
thread.join();
ASSERT_TRUE(called);
}
#endif //! ROCKSDB_LITE
} // namespace rocksdb

Loading…
Cancel
Save