From 7981a43274ab65b0f53eeebb89eb2024209c2d11 Mon Sep 17 00:00:00 2001 From: Yiting Li Date: Thu, 20 Mar 2014 13:42:45 -0700 Subject: [PATCH] Consistency Check Function Summary: Added a function/command to check the consistency of live files' meta data Test Plan: Manual test (size mismatch, file not exist). Command test script. Reviewers: haobo Reviewed By: haobo CC: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D16935 --- HISTORY.md | 2 ++ db/db_impl_readonly.cc | 42 ++++++++++++++++++++++++++++++++++++++++++ include/rocksdb/db.h | 8 ++++++++ tools/ldb_test.py | 25 +++++++++++++++++++++++++ util/ldb_cmd.cc | 26 ++++++++++++++++++++++++++ util/ldb_cmd.h | 16 ++++++++++++++++ util/ldb_tool.cc | 1 + 7 files changed, 120 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index a7713c24c..e1c4a9ee0 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -13,6 +13,8 @@ * Chagned Options.prefix_extractor from raw pointer to shared_ptr (take ownership) Changed HashSkipListRepFactory and HashLinkListRepFactory constructor to not take SliceTransform object (use Options.prefix_extractor implicitly) * Added Env::GetThreadPoolQueueLen(), which returns the waiting queue length of thread pools +* Added DB::CheckConsistency(), which checks the consistency of live files' metadata + Added a corresponding command "checkconsistency" in ldb tool ### New Features * If we find one truncated record at the end of the MANIFEST or WAL files, diff --git a/db/db_impl_readonly.cc b/db/db_impl_readonly.cc index faa2ff3c7..ba780b18f 100644 --- a/db/db_impl_readonly.cc +++ b/db/db_impl_readonly.cc @@ -99,4 +99,46 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname, return s; } +Status DB::CheckConsistency(const Options& options, + const std::string& name) { + DB *db = nullptr; + Status st; + + st = DB::OpenForReadOnly(options, name, &db); + if (!st.ok()) { + return st; + } + + std::vector metadata; + db->GetLiveFilesMetaData(&metadata); + + for (const auto& md : metadata) { + std::string file_path = name + md.name; + + if (!db->GetEnv()->FileExists(file_path)) { + st = Status::Corruption("sst file " + md.name + " doesn't exist"); + break; + } + + uint64_t fsize = 0; + st = db->GetEnv()->GetFileSize(file_path, &fsize); + if (!st.ok()) { + st = Status::Corruption( + "Failed to determine the actual size of file " + md.name + + ": " + st.ToString()); + break; + } + if (fsize != md.size) { + st = Status::Corruption( + "sst file size mismatch: " + md.name + + ". Size recorded in manifest " + std::to_string(md.size) + + ", actual size " + std::to_string(fsize)); + break; + } + } + + delete db; + return st; } + +} // namespace rocksdb diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 1f8d2f37e..a0098b6b0 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -91,6 +91,14 @@ class DB { const std::string& name, DB** dbptr, bool error_if_log_file_exist = false); + // Check the consistency of live files' metadata. + // It will return Corruption Status when a file in manifest + // doesn't actually exist or doesn't match the actual file size. + // Note: This call should be invoked only when the database is + // not already open and serving data. + static Status CheckConsistency(const Options& options, + const std::string& name); + DB() { } virtual ~DB(); diff --git a/tools/ldb_test.py b/tools/ldb_test.py index 4ee3d7fef..b4ef5221f 100644 --- a/tools/ldb_test.py +++ b/tools/ldb_test.py @@ -129,6 +129,8 @@ class LDBTestCase(unittest.TestCase): # It is weird that GET and SCAN raise exception for # non-existent key, while delete does not + self.assertRunOK("checkconsistency", "OK") + def dumpDb(self, params, dumpFile): return 0 == run_err_null("./ldb dump %s > %s" % (params, dumpFile)) @@ -201,6 +203,7 @@ class LDBTestCase(unittest.TestCase): self.assertRunOK("scan", "a1 : b1\na2 : b2\na3 : b3\na4 : b4") self.assertRunOK("delete --hex 0x6133", "OK") self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4") + self.assertRunOK("checkconsistency", "OK") def testTtlPutGet(self): print "Running testTtlPutGet..." @@ -215,6 +218,7 @@ class LDBTestCase(unittest.TestCase): self.assertRunOK("put a3 b3 --create_if_missing", "OK") # fails because timstamp's length is greater than value's self.assertRunFAIL("get --ttl a3") + self.assertRunOK("checkconsistency", "OK") def testInvalidCmdLines(self): print "Running testInvalidCmdLines..." @@ -354,5 +358,26 @@ class LDBTestCase(unittest.TestCase): origDbPath, os.path.join(origDbPath, "LOG")))) self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4") + def testCheckConsistency(self): + print "Running testCheckConsistency..." + + dbPath = os.path.join(self.TMP_DIR, self.DB_NAME) + self.assertRunOK("put x1 y1 --create_if_missing", "OK") + self.assertRunOK("put x2 y2", "OK") + self.assertRunOK("get x1", "y1") + self.assertRunOK("checkconsistency", "OK") + + sstFilePath = my_check_output("ls %s" % os.path.join(dbPath, "*.sst"), + shell=True) + + # Modify the file + my_check_output("echo 'evil' > %s" % sstFilePath, shell=True) + self.assertRunFAIL("checkconsistency") + + # Delete the file + my_check_output("rm -f %s" % sstFilePath, shell=True) + self.assertRunFAIL("checkconsistency") + + if __name__ == "__main__": unittest.main() diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index f373ea18d..8af476323 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -154,6 +154,8 @@ LDBCommand* LDBCommand::SelectCommand( return new ManifestDumpCommand(cmdParams, option_map, flags); } else if (cmd == InternalDumpCommand::Name()) { return new InternalDumpCommand(cmdParams, option_map, flags); + } else if (cmd == CheckConsistencyCommand::Name()) { + return new CheckConsistencyCommand(cmdParams, option_map, flags); } return nullptr; } @@ -1749,5 +1751,29 @@ void DBQuerierCommand::DoCommand() { } } +CheckConsistencyCommand::CheckConsistencyCommand(const vector& params, + const map& options, const vector& flags) : + LDBCommand(options, flags, false, + BuildCmdLineOptions({})) { +} + +void CheckConsistencyCommand::Help(string& ret) { + ret.append(" "); + ret.append(CheckConsistencyCommand::Name()); + ret.append("\n"); +} +void CheckConsistencyCommand::DoCommand() { + Options opt = PrepareOptionsForOpenDB(); + if (!exec_state_.IsNotStarted()) { + return; + } + Status st = DB::CheckConsistency(opt, db_path_); + if (st.ok()) { + fprintf(stdout, "OK\n"); + } else { + exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString()); + } } + +} // namespace rocksdb diff --git a/util/ldb_cmd.h b/util/ldb_cmd.h index 022f5fadc..17343bb18 100644 --- a/util/ldb_cmd.h +++ b/util/ldb_cmd.h @@ -686,4 +686,20 @@ private: static const char* DELETE_CMD; }; +class CheckConsistencyCommand : public LDBCommand { +public: + static string Name() { return "checkconsistency"; } + + CheckConsistencyCommand(const vector& params, + const map& options, const vector& flags); + + virtual void DoCommand(); + + virtual bool NoDBOpen() { + return true; + } + + static void Help(string& ret); +}; + } // namespace rocksdb diff --git a/util/ldb_tool.cc b/util/ldb_tool.cc index 2dbbbf8e0..10d5a1fa1 100644 --- a/util/ldb_tool.cc +++ b/util/ldb_tool.cc @@ -53,6 +53,7 @@ public: DeleteCommand::Help(ret); DBQuerierCommand::Help(ret); ApproxSizeCommand::Help(ret); + CheckConsistencyCommand::Help(ret); ret.append("\n\n"); ret.append("Admin Commands:\n");