Sanity check on Open

Summary:
Everytime a client opens a DB, we do a sanity check that:
* checks the existance of all the necessary files
* verifies that file sizes are correct

Some of the code was stolen from https://reviews.facebook.net/D16935

Test Plan: added a unit test

Reviewers: dhruba, haobo, sdong

Reviewed By: dhruba

CC: leveldb

Differential Revision: https://reviews.facebook.net/D17097
main
Igor Canadi 11 years ago
parent 7981a43274
commit e67241f0b9
  1. 4
      HISTORY.md
  2. 32
      db/corruption_test.cc
  3. 32
      db/db_impl.cc
  4. 4
      db/db_impl.h
  5. 42
      db/db_impl_readonly.cc
  6. 8
      include/rocksdb/db.h
  7. 5
      util/ldb_cmd.cc

@ -13,8 +13,8 @@
* Chagned Options.prefix_extractor from raw pointer to shared_ptr (take ownership)
Changed HashSkipListRepFactory and HashLinkListRepFactory constructor to not take SliceTransform object (use Options.prefix_extractor implicitly)
* Added Env::GetThreadPoolQueueLen(), which returns the waiting queue length of thread pools
* Added DB::CheckConsistency(), which checks the consistency of live files' metadata
Added a corresponding command "checkconsistency" in ldb tool
* Added a command "checkconsistency" in ldb tool, which checks
if file system state matches DB state (file existence and file sizes)
### New Features
* If we find one truncated record at the end of the MANIFEST or WAL files,

@ -376,6 +376,38 @@ TEST(CorruptionTest, UnrelatedKeys) {
ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
}
TEST(CorruptionTest, FileSystemStateCorrupted) {
for (int iter = 0; iter < 2; ++iter) {
Options options;
options.paranoid_checks = true;
options.create_if_missing = true;
Reopen(&options);
Build(10);
ASSERT_OK(db_->Flush(FlushOptions()));
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
std::vector<LiveFileMetaData> metadata;
dbi->GetLiveFilesMetaData(&metadata);
ASSERT_GT(metadata.size(), 0);
std::string filename = dbname_ + metadata[0].name;
delete db_;
if (iter == 0) { // corrupt file size
unique_ptr<WritableFile> file;
env_.NewWritableFile(filename, &file, EnvOptions());
file->Append(Slice("corrupted sst"));
file.reset();
} else { // delete the file
env_.DeleteFile(filename);
}
Status x = TryReopen(&options);
ASSERT_TRUE(x.IsCorruption());
DestroyDB(dbname_, options_);
Reopen(&options);
}
}
} // namespace rocksdb
int main(int argc, char** argv) {

@ -969,6 +969,9 @@ Status DBImpl::Recover(bool read_only, bool error_if_log_file_exist) {
}
Status s = versions_->Recover();
if (options_.paranoid_checks && s.ok()) {
s = CheckConsistency();
}
if (s.ok()) {
SequenceNumber max_sequence(0);
@ -3828,11 +3831,38 @@ Status DBImpl::DeleteFile(std::string name) {
return status;
}
void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata) {
void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
MutexLock l(&mutex_);
return versions_->GetLiveFilesMetaData(metadata);
}
Status DBImpl::CheckConsistency() {
mutex_.AssertHeld();
std::vector<LiveFileMetaData> metadata;
versions_->GetLiveFilesMetaData(&metadata);
std::string corruption_messages;
for (const auto& md : metadata) {
std::string file_path = dbname_ + md.name;
uint64_t fsize = 0;
Status s = env_->GetFileSize(file_path, &fsize);
if (!s.ok()) {
corruption_messages +=
"Can't access " + md.name + ": " + s.ToString() + "\n";
} else if (fsize != md.size) {
corruption_messages += "Sst file size mismatch: " + md.name +
". Size recorded in manifest " +
std::to_string(md.size) + ", actual size " +
std::to_string(fsize) + "\n";
}
}
if (corruption_messages.size() == 0) {
return Status::OK();
} else {
return Status::Corruption(corruption_messages);
}
}
void DBImpl::TEST_GetFilesMetaData(
std::vector<std::vector<FileMetaData>>* metadata) {
MutexLock l(&mutex_);

@ -94,6 +94,10 @@ class DBImpl : public DB {
virtual void GetLiveFilesMetaData(
std::vector<LiveFileMetaData> *metadata);
// checks if all live files exist on file system and that their file sizes
// match to our in-memory records
virtual Status CheckConsistency();
virtual Status GetDbIdentity(std::string& identity);
Status RunManualCompaction(int input_level,

@ -99,46 +99,4 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
return s;
}
Status DB::CheckConsistency(const Options& options,
const std::string& name) {
DB *db = nullptr;
Status st;
st = DB::OpenForReadOnly(options, name, &db);
if (!st.ok()) {
return st;
}
std::vector<LiveFileMetaData> metadata;
db->GetLiveFilesMetaData(&metadata);
for (const auto& md : metadata) {
std::string file_path = name + md.name;
if (!db->GetEnv()->FileExists(file_path)) {
st = Status::Corruption("sst file " + md.name + " doesn't exist");
break;
}
uint64_t fsize = 0;
st = db->GetEnv()->GetFileSize(file_path, &fsize);
if (!st.ok()) {
st = Status::Corruption(
"Failed to determine the actual size of file " + md.name +
": " + st.ToString());
break;
}
if (fsize != md.size) {
st = Status::Corruption(
"sst file size mismatch: " + md.name +
". Size recorded in manifest " + std::to_string(md.size) +
", actual size " + std::to_string(fsize));
break;
}
}
delete db;
return st;
}
} // namespace rocksdb

@ -91,14 +91,6 @@ class DB {
const std::string& name, DB** dbptr,
bool error_if_log_file_exist = false);
// Check the consistency of live files' metadata.
// It will return Corruption Status when a file in manifest
// doesn't actually exist or doesn't match the actual file size.
// Note: This call should be invoked only when the database is
// not already open and serving data.
static Status CheckConsistency(const Options& options,
const std::string& name);
DB() { }
virtual ~DB();

@ -1765,10 +1765,13 @@ void CheckConsistencyCommand::Help(string& ret) {
void CheckConsistencyCommand::DoCommand() {
Options opt = PrepareOptionsForOpenDB();
opt.paranoid_checks = true;
if (!exec_state_.IsNotStarted()) {
return;
}
Status st = DB::CheckConsistency(opt, db_path_);
DB* db;
Status st = DB::OpenForReadOnly(opt, db_path_, &db, false);
delete db;
if (st.ok()) {
fprintf(stdout, "OK\n");
} else {

Loading…
Cancel
Save