NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Enforce status check for corruption_test (#7453)

Browse Source 
Summary:
Enforce status check for corruption_test.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7453

Test Plan:
```
ASSERT_STATUS_CHECKED=1 make corruption_test
./corruption_test
```

Reviewed By: jay-zhuang

Differential Revision: D24006862

Pulled By: riversand963

fbshipit-source-id: 664677caf4c3007a25cf565cec3d677f2dcea130
main
Yanqin Jin 4 years ago committed by Facebook GitHub Bot
parent 7d503e66a9
commit 758ead5df7
10 changed files with 71 additions and 49 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      Makefile
  2. 12
      db/builder.cc
  3. 58
      db/corruption_test.cc
  4. 2
      db/db_impl/db_impl_open.cc
  5. 6
      db/db_impl/db_impl_write.cc
  6. 12
      db/version_builder.cc
  7. 13
      db/version_set.cc
  8. 10
      db/write_thread.cc
  9. 2
      db/write_thread.h
  10. 3
      table/table_factory.cc

2
Makefile
Unescape View File

@ -659,6 +659,8 @@ ifdef ASSERT_STATUS_CHECKED
full_filter_block_test \ full_filter_block_test \
partitioned_filter_block_test \ partitioned_filter_block_test \
column_family_test \ column_family_test \
file_reader_writer_test \
corruption_test \
ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1) ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
TESTS_PASSING_ASC += folly_synchronization_distributed_mutex_test TESTS_PASSING_ASC += folly_synchronization_distributed_mutex_test

12
db/builder.cc
Unescape View File

@ -207,6 +207,11 @@ Status BuildTable(
ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written)); ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
} }
} }
if (!s.ok()) {
c_iter.status().PermitUncheckedError();
} else if (!c_iter.status().ok()) {
s = c_iter.status();
}
if (s.ok()) { if (s.ok()) {
auto range_del_it = range_del_agg->NewIterator(); auto range_del_it = range_del_agg->NewIterator();
for (range_del_it->SeekToFirst(); range_del_it->Valid(); for (range_del_it->SeekToFirst(); range_del_it->Valid();
@ -218,13 +223,8 @@ Status BuildTable(
tombstone.seq_, internal_comparator); tombstone.seq_, internal_comparator);
} }
// Finish and check for builder errors
s = c_iter.status();
if (blob_file_builder) { if (blob_file_builder) {
if (s.ok()) { s = blob_file_builder->Finish();
s = blob_file_builder->Finish();
}
} }
} }

58
db/corruption_test.cc
Unescape View File

@ -56,7 +56,8 @@ class CorruptionTest : public testing::Test {
options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords; options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
options_.env = &env_; options_.env = &env_;
dbname_ = test::PerThreadDBPath("corruption_test"); dbname_ = test::PerThreadDBPath("corruption_test");
DestroyDB(dbname_, options_); Status s = DestroyDB(dbname_, options_);
EXPECT_OK(s);
db_ = nullptr; db_ = nullptr;
options_.create_if_missing = true; options_.create_if_missing = true;
@ -111,12 +112,12 @@ class CorruptionTest : public testing::Test {
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
if (flush_every != 0 && i != 0 && i % flush_every == 0) { if (flush_every != 0 && i != 0 && i % flush_every == 0) {
DBImpl* dbi = static_cast_with_check<DBImpl>(db_); DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
} }
//if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n); //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
Slice key = Key(i, &key_space); Slice key = Key(i, &key_space);
batch.Clear(); batch.Clear();
batch.Put(key, Value(i, &value_space)); ASSERT_OK(batch.Put(key, Value(i, &value_space)));
ASSERT_OK(db_->Write(WriteOptions(), &batch)); ASSERT_OK(db_->Write(WriteOptions(), &batch));
} }
} }
@ -135,6 +136,7 @@ class CorruptionTest : public testing::Test {
// occurred. // occurred.
Iterator* iter = db_->NewIterator(ReadOptions(false, true)); Iterator* iter = db_->NewIterator(ReadOptions(false, true));
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ASSERT_OK(iter->status());
uint64_t key; uint64_t key;
Slice in(iter->key()); Slice in(iter->key());
if (!ConsumeDecimalNumber(&in, &key) || if (!ConsumeDecimalNumber(&in, &key) ||
@ -151,6 +153,7 @@ class CorruptionTest : public testing::Test {
correct++; correct++;
} }
} }
iter->status().PermitUncheckedError();
delete iter; delete iter;
fprintf(stderr, fprintf(stderr,
@ -270,7 +273,7 @@ TEST_F(CorruptionTest, NewFileErrorDuringWrite) {
bool failed = false; bool failed = false;
for (int i = 0; i < num; i++) { for (int i = 0; i < num; i++) {
WriteBatch batch; WriteBatch batch;
batch.Put("a", Value(100, &value_storage)); ASSERT_OK(batch.Put("a", Value(100, &value_storage)));
s = db_->Write(WriteOptions(), &batch); s = db_->Write(WriteOptions(), &batch);
if (!s.ok()) { if (!s.ok()) {
failed = true; failed = true;
@ -286,9 +289,9 @@ TEST_F(CorruptionTest, NewFileErrorDuringWrite) {
TEST_F(CorruptionTest, TableFile) { TEST_F(CorruptionTest, TableFile) {
Build(100); Build(100);
DBImpl* dbi = static_cast_with_check<DBImpl>(db_); DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
dbi->TEST_CompactRange(0, nullptr, nullptr); ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr));
dbi->TEST_CompactRange(1, nullptr, nullptr); ASSERT_OK(dbi->TEST_CompactRange(1, nullptr, nullptr));
Corrupt(kTableFile, 100, 1); Corrupt(kTableFile, 100, 1);
Check(99, 99); Check(99, 99);
@ -309,9 +312,9 @@ TEST_F(CorruptionTest, VerifyChecksumReadahead) {
Build(10000); Build(10000);
DBImpl* dbi = static_cast_with_check<DBImpl>(db_); DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
dbi->TEST_CompactRange(0, nullptr, nullptr); ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr));
dbi->TEST_CompactRange(1, nullptr, nullptr); ASSERT_OK(dbi->TEST_CompactRange(1, nullptr, nullptr));
senv.count_random_reads_ = true; senv.count_random_reads_ = true;
senv.random_read_counter_.Reset(); senv.random_read_counter_.Reset();
@ -356,7 +359,7 @@ TEST_F(CorruptionTest, TableFileIndexData) {
// build 2 tables, flush at 5000 // build 2 tables, flush at 5000
Build(10000, 5000); Build(10000, 5000);
DBImpl* dbi = static_cast_with_check<DBImpl>(db_); DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
// corrupt an index block of an entire file // corrupt an index block of an entire file
Corrupt(kTableFile, -2000, 500); Corrupt(kTableFile, -2000, 500);
@ -403,8 +406,8 @@ TEST_F(CorruptionTest, SequenceNumberRecovery) {
TEST_F(CorruptionTest, CorruptedDescriptor) { TEST_F(CorruptionTest, CorruptedDescriptor) {
ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello")); ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
DBImpl* dbi = static_cast_with_check<DBImpl>(db_); DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
dbi->TEST_CompactRange(0, nullptr, nullptr); ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr));
Corrupt(kDescriptorFile, 0, 1000); Corrupt(kDescriptorFile, 0, 1000);
Status s = TryReopen(); Status s = TryReopen();
@ -422,9 +425,9 @@ TEST_F(CorruptionTest, CompactionInputError) {
Reopen(&options); Reopen(&options);
Build(10); Build(10);
DBImpl* dbi = static_cast_with_check<DBImpl>(db_); DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
dbi->TEST_CompactRange(0, nullptr, nullptr); ASSERT_OK(dbi->TEST_CompactRange(0, nullptr, nullptr));
dbi->TEST_CompactRange(1, nullptr, nullptr); ASSERT_OK(dbi->TEST_CompactRange(1, nullptr, nullptr));
ASSERT_EQ(1, Property("rocksdb.num-files-at-level2")); ASSERT_EQ(1, Property("rocksdb.num-files-at-level2"));
Corrupt(kTableFile, 100, 1); Corrupt(kTableFile, 100, 1);
@ -447,12 +450,12 @@ TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
// Fill levels >= 1 // Fill levels >= 1
for (int level = 1; level < dbi->NumberLevels(); level++) { for (int level = 1; level < dbi->NumberLevels(); level++) {
dbi->Put(WriteOptions(), "", "begin"); ASSERT_OK(dbi->Put(WriteOptions(), "", "begin"));
dbi->Put(WriteOptions(), "~", "end"); ASSERT_OK(dbi->Put(WriteOptions(), "~", "end"));
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
for (int comp_level = 0; comp_level < dbi->NumberLevels() - level; for (int comp_level = 0; comp_level < dbi->NumberLevels() - level;
++comp_level) { ++comp_level) {
dbi->TEST_CompactRange(comp_level, nullptr, nullptr); ASSERT_OK(dbi->TEST_CompactRange(comp_level, nullptr, nullptr));
} }
} }
@ -460,8 +463,8 @@ TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
dbi = static_cast_with_check<DBImpl>(db_); dbi = static_cast_with_check<DBImpl>(db_);
Build(10); Build(10);
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
dbi->TEST_WaitForCompact(); ASSERT_OK(dbi->TEST_WaitForCompact());
ASSERT_EQ(1, Property("rocksdb.num-files-at-level0")); ASSERT_EQ(1, Property("rocksdb.num-files-at-level0"));
CorruptTableFileAtLevel(0, 100, 1); CorruptTableFileAtLevel(0, 100, 1);
@ -486,7 +489,7 @@ TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
TEST_F(CorruptionTest, UnrelatedKeys) { TEST_F(CorruptionTest, UnrelatedKeys) {
Build(10); Build(10);
DBImpl* dbi = static_cast_with_check<DBImpl>(db_); DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
Corrupt(kTableFile, 100, 1); Corrupt(kTableFile, 100, 1);
ASSERT_NOK(dbi->VerifyChecksum()); ASSERT_NOK(dbi->VerifyChecksum());
@ -495,7 +498,7 @@ TEST_F(CorruptionTest, UnrelatedKeys) {
std::string v; std::string v;
ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
ASSERT_EQ(Value(1000, &tmp2).ToString(), v); ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
dbi->TEST_FlushMemTable(); ASSERT_OK(dbi->TEST_FlushMemTable());
ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
ASSERT_EQ(Value(1000, &tmp2).ToString(), v); ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
} }
@ -548,17 +551,17 @@ TEST_F(CorruptionTest, FileSystemStateCorrupted) {
if (iter == 0) { // corrupt file size if (iter == 0) { // corrupt file size
std::unique_ptr<WritableFile> file; std::unique_ptr<WritableFile> file;
env_.NewWritableFile(filename, &file, EnvOptions()); env_.NewWritableFile(filename, &file, EnvOptions());
file->Append(Slice("corrupted sst")); ASSERT_OK(file->Append(Slice("corrupted sst")));
file.reset(); file.reset();
Status x = TryReopen(&options); Status x = TryReopen(&options);
ASSERT_TRUE(x.IsCorruption()); ASSERT_TRUE(x.IsCorruption());
} else { // delete the file } else { // delete the file
env_.DeleteFile(filename); ASSERT_OK(env_.DeleteFile(filename));
Status x = TryReopen(&options); Status x = TryReopen(&options);
ASSERT_TRUE(x.IsPathNotFound()); ASSERT_TRUE(x.IsPathNotFound());
} }
DestroyDB(dbname_, options_); ASSERT_OK(DestroyDB(dbname_, options_));
} }
} }
@ -577,6 +580,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnFlush) {
delete db_; delete db_;
db_ = nullptr; db_ = nullptr;
s = DestroyDB(dbname_, options); s = DestroyDB(dbname_, options);
ASSERT_OK(s);
std::shared_ptr<mock::MockTableFactory> mock = std::shared_ptr<mock::MockTableFactory> mock =
std::make_shared<mock::MockTableFactory>(); std::make_shared<mock::MockTableFactory>();
options.table_factory = mock; options.table_factory = mock;

2
db/db_impl/db_impl_open.cc
Unescape View File

@ -1678,6 +1678,8 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
*dbptr = impl; *dbptr = impl;
impl->opened_successfully_ = true; impl->opened_successfully_ = true;
impl->MaybeScheduleFlushOrCompaction(); impl->MaybeScheduleFlushOrCompaction();
} else {
persist_options_status.PermitUncheckedError();
} }
impl->mutex_.Unlock(); impl->mutex_.Unlock();

6
db/db_impl/db_impl_write.cc
Unescape View File

@ -999,8 +999,10 @@ WriteBatch* DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group,
merged_batch = tmp_batch; merged_batch = tmp_batch;
for (auto writer : write_group) { for (auto writer : write_group) {
if (!writer->CallbackFailed()) { if (!writer->CallbackFailed()) {
WriteBatchInternal::Append(merged_batch, writer->batch, Status s = WriteBatchInternal::Append(merged_batch, writer->batch,
/*WAL_only*/ true); /*WAL_only*/ true);
// Always returns Status::OK.
assert(s.ok());
if (WriteBatchInternal::IsLatestPersistentState(writer->batch)) { if (WriteBatchInternal::IsLatestPersistentState(writer->batch)) {
// We only need to cache the last of such write batch // We only need to cache the last of such write batch
*to_be_cached_state = writer->batch; *to_be_cached_state = writer->batch;

12
db/version_builder.cc
Unescape View File

@ -979,17 +979,15 @@ class VersionBuilder::Rep {
for (auto& t : threads) { for (auto& t : threads) {
t.join(); t.join();
} }
#ifdef ROCKSDB_ASSERT_STATUS_CHECKED Status ret;
for (const auto& s : statuses) {
s.PermitUncheckedError();
}
#endif // ROCKSDB_ASSERT_STATUS_CHECKED
for (const auto& s : statuses) { for (const auto& s : statuses) {
if (!s.ok()) { if (!s.ok()) {
return s; if (ret.ok()) {
ret = s;
}
} }
} }
return Status::OK(); return ret;
} }
void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f) { void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f) {

13
db/version_set.cc
Unescape View File

@ -4132,10 +4132,15 @@ Status VersionSet::ProcessManifestWrites(
ROCKS_LOG_INFO(db_options_->info_log, ROCKS_LOG_INFO(db_options_->info_log,
"Deleting manifest %" PRIu64 " current manifest %" PRIu64 "Deleting manifest %" PRIu64 " current manifest %" PRIu64
"\n", "\n",
manifest_file_number_, pending_manifest_file_number_); pending_manifest_file_number_, manifest_file_number_);
env_->DeleteFile( Status manifest_del_status = env_->DeleteFile(
DescriptorFileName(dbname_, pending_manifest_file_number_)) DescriptorFileName(dbname_, pending_manifest_file_number_));
.PermitUncheckedError(); if (!manifest_del_status.ok()) {
ROCKS_LOG_WARN(db_options_->info_log,
"Failed to delete manifest %" PRIu64 ": %s",
pending_manifest_file_number_,
manifest_del_status.ToString().c_str());
}
} }
} }

10
db/write_thread.cc
Unescape View File

@ -602,6 +602,8 @@ bool WriteThread::CompleteParallelMemTableWriter(Writer* w) {
} }
// else we're the last parallel worker and should perform exit duties. // else we're the last parallel worker and should perform exit duties.
w->status = write_group->status; w->status = write_group->status;
// Callers of this function must ensure w->status is checked.
write_group->status.PermitUncheckedError();
return true; return true;
} }
@ -618,11 +620,17 @@ void WriteThread::ExitAsBatchGroupFollower(Writer* w) {
static WriteThread::AdaptationContext eabgl_ctx("ExitAsBatchGroupLeader"); static WriteThread::AdaptationContext eabgl_ctx("ExitAsBatchGroupLeader");
void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group, void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group,
Status status) { Status& status) {
Writer* leader = write_group.leader; Writer* leader = write_group.leader;
Writer* last_writer = write_group.last_writer; Writer* last_writer = write_group.last_writer;
assert(leader->link_older == nullptr); assert(leader->link_older == nullptr);
// If status is non-ok already, then write_group.status won't have the chance
// of being propagated to caller.
if (!status.ok()) {
write_group.status.PermitUncheckedError();
}
// Propagate memtable write error to the whole group. // Propagate memtable write error to the whole group.
if (status.ok() && !write_group.status.ok()) { if (status.ok() && !write_group.status.ok()) {
status = write_group.status; status = write_group.status;

2
db/write_thread.h
Unescape View File

@ -292,7 +292,7 @@ class WriteThread {
// //
// WriteGroup* write_group: the write group // WriteGroup* write_group: the write group
// Status status: Status of write operation // Status status: Status of write operation
void ExitAsBatchGroupLeader(WriteGroup& write_group, Status status); void ExitAsBatchGroupLeader(WriteGroup& write_group, Status& status);
// Exit batch group on behalf of batch group leader. // Exit batch group on behalf of batch group leader.
void ExitAsBatchGroupFollower(Writer* w); void ExitAsBatchGroupFollower(Writer* w);

3
table/table_factory.cc
Unescape View File

@ -37,7 +37,8 @@ Status TableFactory::CreateFromString(const ConfigOptions& config_options_in,
factory->reset(new CuckooTableFactory()); factory->reset(new CuckooTableFactory());
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE
} else { } else {
return Status::NotSupported("Could not load table factory: ", name); status = Status::NotSupported("Could not load table factory: ", name);
return status;
} }
if (!existing_opts.empty()) { if (!existing_opts.empty()) {
config_options.invoke_prepare_options = false; config_options.invoke_prepare_options = false;

Write Preview
Loading…
Cancel
Save