Fail fast in paranoid mode when LoadTableHandlers fail during recovering (#6368)

Summary:
Previously, when recovering version set, LoadTableHandlers failures are ignored.
If paranoid_checks is true, this failure should not be ignored, otherwise, the opened db might be in an inconsistent state.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6368

Test Plan: make check

Differential Revision: D19713459

Pulled By: cheng-chang

fbshipit-source-id: 68cb94f4f2cc43f8b024b14755193cd45cfcad55
main
Cheng Chang 5 years ago committed by Facebook Github Bot
parent 29e24434fe
commit 4034e289ad
  1. 21
      db/corruption_test.cc
  2. 16
      db/cuckoo_table_db_test.cc
  3. 14
      db/plain_table_db_test.cc
  4. 24
      db/version_set.cc

@ -393,12 +393,16 @@ TEST_F(CorruptionTest, TableFileIndexData) {
// corrupt an index block of an entire file // corrupt an index block of an entire file
Corrupt(kTableFile, -2000, 500); Corrupt(kTableFile, -2000, 500);
Reopen(); options.paranoid_checks = false;
Reopen(&options);
dbi = reinterpret_cast<DBImpl*>(db_); dbi = reinterpret_cast<DBImpl*>(db_);
// one full file may be readable, since only one was corrupted // one full file may be readable, since only one was corrupted
// the other file should be fully non-readable, since index was corrupted // the other file should be fully non-readable, since index was corrupted
Check(0, 5000); Check(0, 5000);
ASSERT_NOK(dbi->VerifyChecksum()); ASSERT_NOK(dbi->VerifyChecksum());
// In paranoid mode, the db cannot be opened due to the corrupted file.
ASSERT_TRUE(TryReopen().IsCorruption());
} }
TEST_F(CorruptionTest, MissingDescriptor) { TEST_F(CorruptionTest, MissingDescriptor) {
@ -554,13 +558,7 @@ TEST_F(CorruptionTest, RangeDeletionCorrupted) {
ASSERT_OK(TryReopen()); ASSERT_OK(TryReopen());
CorruptFile(filename, static_cast<int>(range_del_handle.offset()), 1); CorruptFile(filename, static_cast<int>(range_del_handle.offset()), 1);
// The test case does not fail on TryReopen because failure to preload table ASSERT_TRUE(TryReopen().IsCorruption());
// handlers is not considered critical.
ASSERT_OK(TryReopen());
std::string val;
// However, it does fail on any read involving that file since that file
// cannot be opened with a corrupt range deletion meta-block.
ASSERT_TRUE(db_->Get(ReadOptions(), "a", &val).IsCorruption());
} }
TEST_F(CorruptionTest, FileSystemStateCorrupted) { TEST_F(CorruptionTest, FileSystemStateCorrupted) {
@ -585,14 +583,15 @@ TEST_F(CorruptionTest, FileSystemStateCorrupted) {
env_.NewWritableFile(filename, &file, EnvOptions()); env_.NewWritableFile(filename, &file, EnvOptions());
file->Append(Slice("corrupted sst")); file->Append(Slice("corrupted sst"));
file.reset(); file.reset();
Status x = TryReopen(&options);
ASSERT_TRUE(x.IsCorruption());
} else { // delete the file } else { // delete the file
env_.DeleteFile(filename); env_.DeleteFile(filename);
Status x = TryReopen(&options);
ASSERT_TRUE(x.IsPathNotFound());
} }
Status x = TryReopen(&options);
ASSERT_TRUE(x.IsCorruption());
DestroyDB(dbname_, options_); DestroyDB(dbname_, options_);
Reopen(&options);
} }
} }

@ -298,17 +298,25 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) {
dbfull()->TEST_FlushMemTable(); dbfull()->TEST_FlushMemTable();
// Write some keys using plain table. // Write some keys using plain table.
std::shared_ptr<TableFactory> block_based_factory(
NewBlockBasedTableFactory());
std::shared_ptr<TableFactory> plain_table_factory(
NewPlainTableFactory());
std::shared_ptr<TableFactory> cuckoo_table_factory(
NewCuckooTableFactory());
options.create_if_missing = false; options.create_if_missing = false;
options.table_factory.reset(NewPlainTableFactory()); options.table_factory.reset(NewAdaptiveTableFactory(
plain_table_factory, block_based_factory, plain_table_factory,
cuckoo_table_factory));
Reopen(&options); Reopen(&options);
ASSERT_OK(Put("key4", "v4")); ASSERT_OK(Put("key4", "v4"));
ASSERT_OK(Put("key1", "v5")); ASSERT_OK(Put("key1", "v5"));
dbfull()->TEST_FlushMemTable(); dbfull()->TEST_FlushMemTable();
// Write some keys using block based table. // Write some keys using block based table.
std::shared_ptr<TableFactory> block_based_factory( options.table_factory.reset(NewAdaptiveTableFactory(
NewBlockBasedTableFactory()); block_based_factory, block_based_factory, plain_table_factory,
options.table_factory.reset(NewAdaptiveTableFactory(block_based_factory)); cuckoo_table_factory));
Reopen(&options); Reopen(&options);
ASSERT_OK(Put("key5", "v6")); ASSERT_OK(Put("key5", "v6"));
ASSERT_OK(Put("key2", "v7")); ASSERT_OK(Put("key2", "v7"));

@ -401,19 +401,17 @@ TEST_P(PlainTableDBTest, BadOptions1) {
// Bad attempt to re-open without a prefix extractor // Bad attempt to re-open without a prefix extractor
Options options = CurrentOptions(); Options options = CurrentOptions();
options.prefix_extractor.reset(); options.prefix_extractor.reset();
Reopen(&options);
ASSERT_EQ( ASSERT_EQ(
"Invalid argument: Prefix extractor is missing when opening a PlainTable " "Invalid argument: Prefix extractor is missing when opening a PlainTable "
"built using a prefix extractor", "built using a prefix extractor",
Get("1000000000000foo")); TryReopen(&options).ToString());
// Bad attempt to re-open with different prefix extractor // Bad attempt to re-open with different prefix extractor
options.prefix_extractor.reset(NewFixedPrefixTransform(6)); options.prefix_extractor.reset(NewFixedPrefixTransform(6));
Reopen(&options);
ASSERT_EQ( ASSERT_EQ(
"Invalid argument: Prefix extractor given doesn't match the one used to " "Invalid argument: Prefix extractor given doesn't match the one used to "
"build PlainTable", "build PlainTable",
Get("1000000000000foo")); TryReopen(&options).ToString());
// Correct prefix extractor // Correct prefix extractor
options.prefix_extractor.reset(NewFixedPrefixTransform(8)); options.prefix_extractor.reset(NewFixedPrefixTransform(8));
@ -1323,11 +1321,13 @@ TEST_P(PlainTableDBTest, AdaptiveTable) {
dbfull()->TEST_FlushMemTable(); dbfull()->TEST_FlushMemTable();
options.create_if_missing = false; options.create_if_missing = false;
std::shared_ptr<TableFactory> dummy_factory;
std::shared_ptr<TableFactory> block_based_factory( std::shared_ptr<TableFactory> block_based_factory(
NewBlockBasedTableFactory()); NewBlockBasedTableFactory());
std::shared_ptr<TableFactory> plain_table_factory(
NewPlainTableFactory());
std::shared_ptr<TableFactory> dummy_factory;
options.table_factory.reset(NewAdaptiveTableFactory( options.table_factory.reset(NewAdaptiveTableFactory(
block_based_factory, dummy_factory, dummy_factory)); block_based_factory, block_based_factory, plain_table_factory));
Reopen(&options); Reopen(&options);
ASSERT_EQ("v3", Get("1000000000000foo")); ASSERT_EQ("v3", Get("1000000000000foo"));
ASSERT_EQ("v2", Get("0000000000000bar")); ASSERT_EQ("v2", Get("0000000000000bar"));
@ -1344,10 +1344,12 @@ TEST_P(PlainTableDBTest, AdaptiveTable) {
ASSERT_EQ("v4", Get("2000000000000foo")); ASSERT_EQ("v4", Get("2000000000000foo"));
ASSERT_EQ("v5", Get("3000000000000bar")); ASSERT_EQ("v5", Get("3000000000000bar"));
options.paranoid_checks = false;
options.table_factory.reset(NewBlockBasedTableFactory()); options.table_factory.reset(NewBlockBasedTableFactory());
Reopen(&options); Reopen(&options);
ASSERT_NE("v3", Get("1000000000000foo")); ASSERT_NE("v3", Get("1000000000000foo"));
options.paranoid_checks = false;
options.table_factory.reset(NewPlainTableFactory()); options.table_factory.reset(NewPlainTableFactory());
Reopen(&options); Reopen(&options);
ASSERT_NE("v5", Get("3000000000000bar")); ASSERT_NE("v5", Get("3000000000000bar"));

@ -3803,17 +3803,23 @@ Status VersionSet::ProcessManifestWrites(
assert(!mutable_cf_options_ptrs.empty() && assert(!mutable_cf_options_ptrs.empty() &&
builder_guards.size() == versions.size()); builder_guards.size() == versions.size());
ColumnFamilyData* cfd = versions[i]->cfd_; ColumnFamilyData* cfd = versions[i]->cfd_;
builder_guards[i]->version_builder()->LoadTableHandlers( s = builder_guards[i]->version_builder()->LoadTableHandlers(
cfd->internal_stats(), cfd->ioptions()->optimize_filters_for_hits, cfd->internal_stats(), cfd->ioptions()->optimize_filters_for_hits,
true /* prefetch_index_and_filter_in_cache */, true /* prefetch_index_and_filter_in_cache */,
false /* is_initial_load */, false /* is_initial_load */,
mutable_cf_options_ptrs[i]->prefix_extractor.get()); mutable_cf_options_ptrs[i]->prefix_extractor.get());
if (!s.ok()) {
if (db_options_->paranoid_checks) {
break;
}
s = Status::OK();
}
} }
} }
if (s.ok() && new_descriptor_log) {
// This is fine because everything inside of this block is serialized -- // This is fine because everything inside of this block is serialized --
// only one thread can be here at the same time // only one thread can be here at the same time
if (new_descriptor_log) {
// create new manifest file // create new manifest file
ROCKS_LOG_INFO(db_options_->info_log, "Creating manifest %" PRIu64 "\n", ROCKS_LOG_INFO(db_options_->info_log, "Creating manifest %" PRIu64 "\n",
pending_manifest_file_number_); pending_manifest_file_number_);
@ -3835,6 +3841,7 @@ Status VersionSet::ProcessManifestWrites(
} }
} }
if (s.ok()) {
if (!first_writer.edit_list.front()->IsColumnFamilyManipulation()) { if (!first_writer.edit_list.front()->IsColumnFamilyManipulation()) {
for (int i = 0; i < static_cast<int>(versions.size()); ++i) { for (int i = 0; i < static_cast<int>(versions.size()); ++i) {
versions[i]->PrepareApply(*mutable_cf_options_ptrs[i], true); versions[i]->PrepareApply(*mutable_cf_options_ptrs[i], true);
@ -3842,7 +3849,6 @@ Status VersionSet::ProcessManifestWrites(
} }
// Write new records to MANIFEST log // Write new records to MANIFEST log
if (s.ok()) {
#ifndef NDEBUG #ifndef NDEBUG
size_t idx = 0; size_t idx = 0;
#endif #endif
@ -4386,8 +4392,8 @@ Status VersionSet::Recover(
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only, const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
std::string* db_id) { std::string* db_id) {
std::unordered_map<std::string, ColumnFamilyOptions> cf_name_to_options; std::unordered_map<std::string, ColumnFamilyOptions> cf_name_to_options;
for (auto cf : column_families) { for (const auto& cf : column_families) {
cf_name_to_options.insert({cf.name, cf.options}); cf_name_to_options.emplace(cf.name, cf.options);
} }
// keeps track of column families in manifest that were not found in // keeps track of column families in manifest that were not found in
// column families parameters. if those column families are not dropped // column families parameters. if those column families are not dropped
@ -4518,11 +4524,17 @@ Status VersionSet::Recover(
// unlimited table cache. Pre-load table handle now. // unlimited table cache. Pre-load table handle now.
// Need to do it out of the mutex. // Need to do it out of the mutex.
builder->LoadTableHandlers( s = builder->LoadTableHandlers(
cfd->internal_stats(), db_options_->max_file_opening_threads, cfd->internal_stats(), db_options_->max_file_opening_threads,
false /* prefetch_index_and_filter_in_cache */, false /* prefetch_index_and_filter_in_cache */,
true /* is_initial_load */, true /* is_initial_load */,
cfd->GetLatestMutableCFOptions()->prefix_extractor.get()); cfd->GetLatestMutableCFOptions()->prefix_extractor.get());
if (!s.ok()) {
if (db_options_->paranoid_checks) {
return s;
}
s = Status::OK();
}
Version* v = new Version(cfd, this, file_options_, Version* v = new Version(cfd, this, file_options_,
*cfd->GetLatestMutableCFOptions(), *cfd->GetLatestMutableCFOptions(),

Loading…
Cancel
Save