From 20ee76bd3464c3ba10d962062f3901d1b54dfdb8 Mon Sep 17 00:00:00 2001 From: heyongqiang Date: Fri, 17 Aug 2012 16:06:05 -0700 Subject: [PATCH] use ts as suffix for LOG.old files Summary: as subject and only maintain 10 log files. Test Plan: new test in db_test Reviewers: dhruba Differential Revision: https://reviews.facebook.net/D4731 --- db/db_impl.cc | 26 ++++++++++++++++++++++++-- db/db_impl.h | 2 ++ db/db_test.cc | 16 ++++++++++++++++ db/filename.cc | 18 ++++++++++++++---- db/filename.h | 2 +- 5 files changed, 57 insertions(+), 7 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index 72e85f936..f011ed76f 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include "db/builder.h" #include "db/db_iter.h" #include "db/dbformat.h" @@ -99,7 +100,8 @@ Options SanitizeOptions(const std::string& dbname, if (result.info_log == NULL) { // Open a log file in the same directory as the db src.env->CreateDir(dbname); // In case it does not exist - src.env->RenameFile(InfoLogFileName(dbname), OldInfoLogFileName(dbname)); + src.env->RenameFile(InfoLogFileName(dbname), + OldInfoLogFileName(dbname, src.env->NowMicros())); Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log); if (!s.ok()) { // No place suitable for logging @@ -224,6 +226,7 @@ void DBImpl::DeleteObsoleteFiles() { env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose uint64_t number; FileType type; + std::vector old_log_files_ts; for (size_t i = 0; i < filenames.size(); i++) { if (ParseFileName(filenames[i], &number, &type)) { bool keep = true; @@ -245,9 +248,14 @@ void DBImpl::DeleteObsoleteFiles() { // be recorded in pending_outputs_, which is inserted into "live" keep = (live.find(number) != live.end()); break; + case kInfoLogFile: + keep = true; + if (number != 0) { + old_log_files_ts.push_back(number); + } + break; case kCurrentFile: case kDBLockFile: - case kInfoLogFile: keep = true; break; } @@ -263,6 +271,20 @@ void DBImpl::DeleteObsoleteFiles() { } } } + + // Delete old log files. + int old_log_file_count = old_log_files_ts.size(); + if (old_log_file_count >= KEEP_LOG_FILE_NUM) { + std::sort(old_log_files_ts.begin(), old_log_files_ts.end()); + for (int i = 0; i >= (old_log_file_count - KEEP_LOG_FILE_NUM); i++) { + uint64_t ts = old_log_files_ts.at(i); + std::string to_delete = OldInfoLogFileName(dbname_, ts); + Log(options_.info_log, "Delete type=%d #%lld\n", + int(kInfoLogFile), + static_cast(ts)); + env_->DeleteFile(dbname_ + "/" + to_delete); + } + } } Status DBImpl::Recover(VersionEdit* edit) { diff --git a/db/db_impl.h b/db/db_impl.h index 9ff82caf5..cf70a1b98 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -189,6 +189,8 @@ class DBImpl : public DB { }; CompactionStats* stats_; + static const int KEEP_LOG_FILE_NUM = 1000; + // No copying allowed DBImpl(const DBImpl&); void operator=(const DBImpl&); diff --git a/db/db_test.cc b/db/db_test.cc index 105a8879b..e2abdf9b0 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -779,6 +779,22 @@ TEST(DBTest, Recover) { } while (ChangeOptions()); } +TEST(DBTest, RollLog) { + do { + ASSERT_OK(Put("foo", "v1")); + ASSERT_OK(Put("baz", "v5")); + + Reopen(); + for (int i = 0; i < 10; i++) { + Reopen(); + } + ASSERT_OK(Put("foo", "v4")); + for (int i = 0; i < 10; i++) { + Reopen(); + } + } while (ChangeOptions()); +} + TEST(DBTest, WAL) { Options options = CurrentOptions(); WriteOptions writeOpt = WriteOptions(); diff --git a/db/filename.cc b/db/filename.cc index 3c4d49f64..cc9bc5994 100644 --- a/db/filename.cc +++ b/db/filename.cc @@ -60,8 +60,10 @@ std::string InfoLogFileName(const std::string& dbname) { } // Return the name of the old info log file for "dbname". -std::string OldInfoLogFileName(const std::string& dbname) { - return dbname + "/LOG.old"; +std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts) { + char buf[50]; + snprintf(buf, sizeof(buf), "%llu", static_cast(ts)); + return dbname + "/LOG.old." + buf; } @@ -69,7 +71,7 @@ std::string OldInfoLogFileName(const std::string& dbname) { // dbname/CURRENT // dbname/LOCK // dbname/LOG -// dbname/LOG.old +// dbname/LOG.old.[0-9]+ // dbname/MANIFEST-[0-9]+ // dbname/[0-9]+.(log|sst) bool ParseFileName(const std::string& fname, @@ -82,9 +84,17 @@ bool ParseFileName(const std::string& fname, } else if (rest == "LOCK") { *number = 0; *type = kDBLockFile; - } else if (rest == "LOG" || rest == "LOG.old") { + } else if (rest == "LOG") { *number = 0; *type = kInfoLogFile; + } else if (rest.starts_with("LOG.old.")) { + uint64_t ts_suffix; + rest.remove_prefix(sizeof("LOG.old.")); + if (!ConsumeDecimalNumber(&rest, &ts_suffix)) { + return false; + } + *number = ts_suffix; + *type = kInfoLogFile; } else if (rest.starts_with("MANIFEST-")) { rest.remove_prefix(strlen("MANIFEST-")); uint64_t num; diff --git a/db/filename.h b/db/filename.h index d5d09b114..e6d6b19b5 100644 --- a/db/filename.h +++ b/db/filename.h @@ -60,7 +60,7 @@ extern std::string TempFileName(const std::string& dbname, uint64_t number); extern std::string InfoLogFileName(const std::string& dbname); // Return the name of the old info log file for "dbname". -extern std::string OldInfoLogFileName(const std::string& dbname); +extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts); // If filename is a leveldb file, store the type of the file in *type. // The number encoded in the filename is stored in *number. If the