From 0f43aa474e1a12e2984630b36a8157a59bb5e458 Mon Sep 17 00:00:00 2001 From: heyongqiang Date: Wed, 5 Sep 2012 17:44:13 -0700 Subject: [PATCH] put log in a seperate dir Summary: added a new option db_log_dir, which points the log dir. Inside that dir, in order to make log names unique, the log file name is prefixed with the leveldb data dir absolute path. Test Plan: db_test Reviewers: dhruba Reviewed By: dhruba Differential Revision: https://reviews.facebook.net/D5205 --- db/c.cc | 20 ++++++++++++++++ db/db_impl.cc | 30 +++++++++++++----------- db/db_impl.h | 1 + db/db_stats_logger.cc | 4 +--- db/db_test.cc | 12 ++++++---- db/filename.cc | 49 +++++++++++++++++++++++++++++++++++---- db/filename.h | 6 +++-- include/leveldb/options.h | 8 +++++++ port/port_posix.h | 2 +- util/options.cc | 5 +++- 10 files changed, 109 insertions(+), 28 deletions(-) diff --git a/db/c.cc b/db/c.cc index 62a068416..c775fc919 100644 --- a/db/c.cc +++ b/db/c.cc @@ -502,6 +502,26 @@ void leveldb_options_set_compression(leveldb_options_t* opt, int t) { opt->rep.compression = static_cast(t); } +void leveldb_options_set_disable_data_sync( + leveldb_options_t* opt, bool disable_data_sync) { + opt->rep.disableDataSync = disable_data_sync; +} + +void leveldb_options_set_use_fsync( + leveldb_options_t* opt, bool use_fsync) { + opt->rep.use_fsync = use_fsync; +} + +void leveldb_options_set_db_stats_log_interval( + leveldb_options_t* opt, int db_stats_log_interval) { + opt->rep.db_stats_log_interval = db_stats_log_interval; +} + +void leveldb_options_set_db_log_dir( + leveldb_options_t* opt, const char* db_log_dir) { + opt->rep.db_log_dir = db_log_dir; +} + leveldb_comparator_t* leveldb_comparator_create( void* state, void (*destructor)(void*), diff --git a/db/db_impl.cc b/db/db_impl.cc index af7e51997..18d51baca 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -98,12 +98,16 @@ Options SanitizeOptions(const std::string& dbname, ClipToRange(&result.max_open_files, 20, 50000); ClipToRange(&result.write_buffer_size, 64<<10, 1<<30); ClipToRange(&result.block_size, 1<<10, 4<<20); + std::string db_absolute_path; + src.env->GetAbsolutePath(dbname, &db_absolute_path); if (result.info_log == NULL) { // Open a log file in the same directory as the db src.env->CreateDir(dbname); // In case it does not exist - src.env->RenameFile(InfoLogFileName(dbname), - OldInfoLogFileName(dbname, src.env->NowMicros())); - Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log); + src.env->RenameFile(InfoLogFileName(dbname, db_absolute_path, + result.db_log_dir), OldInfoLogFileName(dbname,src.env->NowMicros(), + db_absolute_path, result.db_log_dir)); + Status s = src.env->NewLogger(InfoLogFileName(dbname, db_absolute_path, + result.db_log_dir), &result.info_log); if (!s.ok()) { // No place suitable for logging result.info_log = NULL; @@ -140,6 +144,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname) mem_->Ref(); has_imm_.Release_Store(NULL); + env_->GetAbsolutePath(dbname, &db_absolute_path_); stats_ = new CompactionStats[options.num_levels]; // Reserve ten files or so for other uses and give the rest to TableCache. const int table_cache_size = options_.max_open_files - 10; @@ -247,7 +252,7 @@ void DBImpl::DeleteObsoleteFiles() { env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose uint64_t number; FileType type; - std::vector old_log_files_ts; + std::vector old_log_files; for (size_t i = 0; i < filenames.size(); i++) { if (ParseFileName(filenames[i], &number, &type)) { bool keep = true; @@ -272,7 +277,7 @@ void DBImpl::DeleteObsoleteFiles() { case kInfoLogFile: keep = true; if (number != 0) { - old_log_files_ts.push_back(number); + old_log_files.push_back(filenames[i]); } break; case kCurrentFile: @@ -299,15 +304,14 @@ void DBImpl::DeleteObsoleteFiles() { } // Delete old log files. - int old_log_file_count = old_log_files_ts.size(); - if (old_log_file_count >= KEEP_LOG_FILE_NUM) { - std::sort(old_log_files_ts.begin(), old_log_files_ts.end()); + int old_log_file_count = old_log_files.size(); + if (old_log_file_count >= KEEP_LOG_FILE_NUM && + !options_.db_log_dir.empty()) { + std::sort(old_log_files.begin(), old_log_files.end()); for (int i = 0; i >= (old_log_file_count - KEEP_LOG_FILE_NUM); i++) { - uint64_t ts = old_log_files_ts.at(i); - std::string to_delete = OldInfoLogFileName(dbname_, ts); - Log(options_.info_log, "Delete type=%d #%lld\n", - int(kInfoLogFile), - static_cast(ts)); + std::string& to_delete = old_log_files.at(i); + Log(options_.info_log, "Delete type=%d %s\n", + int(kInfoLogFile), to_delete.c_str()); env_->DeleteFile(dbname_ + "/" + to_delete); } } diff --git a/db/db_impl.h b/db/db_impl.h index 3f64d58d3..174ac4231 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -210,6 +210,7 @@ class DBImpl : public DB { CompactionStats* stats_; static const int KEEP_LOG_FILE_NUM = 1000; + std::string db_absolute_path_; // No copying allowed DBImpl(const DBImpl&); diff --git a/db/db_stats_logger.cc b/db/db_stats_logger.cc index 7216eef61..aba3ced43 100644 --- a/db/db_stats_logger.cc +++ b/db/db_stats_logger.cc @@ -74,11 +74,9 @@ void DBImpl::LogDBDeployStats() { int64_t unix_ts; env_->GetCurrentTime(&unix_ts); - std::string data_dir; - env_->GetAbsolutePath(dbname_, &data_dir); logger_->Log_Deploy_Stats(version_info, host_name_, - data_dir, file_total_size, file_total_num, file_num_per_level, + db_absolute_path_, file_total_size, file_total_num, file_num_per_level, data_size_per_level, unix_ts); mutex_.Lock(); diff --git a/db/db_test.cc b/db/db_test.cc index 8e2233936..859167822 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -154,6 +154,7 @@ class DBTest { kFilter, kUncompressed, kNumLevel_3, + kDBLogDir, kEnd }; int option_config_; @@ -201,11 +202,14 @@ class DBTest { options.filter_policy = filter_policy_; break; case kUncompressed: - options.compression = kNoCompression; - break; + options.compression = kNoCompression; + break; case kNumLevel_3: - options.num_levels = 3; - break; + options.num_levels = 3; + break; + case kDBLogDir: + options.db_log_dir = test::TmpDir(); + break; default: break; } diff --git a/db/filename.cc b/db/filename.cc index 9fa39e6c7..ad9bb7b4b 100644 --- a/db/filename.cc +++ b/db/filename.cc @@ -8,9 +8,37 @@ #include "db/dbformat.h" #include "leveldb/env.h" #include "util/logging.h" +#include namespace leveldb { +// Given a path, flatten the path name by replacing all chars not in +// {[0-9,a-z,A-Z,-,_,.]} with _. And append '\0' at the end. +// Return the number of chars stored in dest not including the trailing '\0'. +static int FlattenPath(const std::string& path, char* dest, int len) { + int write_idx = 0; + int i = 0; + int src_len = path.size(); + + while (i < src_len && write_idx < len - 1) { + if ((path[i] >= 'a' && path[i] <= 'z') || + (path[i] >= '0' && path[i] <= '9') || + (path[i] >= 'A' && path[i] <= 'Z') || + path[i] == '-' || + path[i] == '.' || + path[i] == '_'){ + dest[write_idx++] = path[i]; + } else { + if (i > 0) + dest[write_idx++] = '_'; + } + i++; + } + + dest[write_idx] = '\0'; + return write_idx; +} + // A utility routine: write "data" to the named file and Sync() it. extern Status WriteStringToFileSync(Env* env, const Slice& data, const std::string& fname); @@ -55,15 +83,28 @@ std::string TempFileName(const std::string& dbname, uint64_t number) { return MakeFileName(dbname, number, "dbtmp"); } -std::string InfoLogFileName(const std::string& dbname) { - return dbname + "/LOG"; +std::string InfoLogFileName(const std::string& dbname, + const std::string& db_path, const std::string& log_dir) { + if (log_dir.empty()) + return dbname + "/LOG"; + + char flatten_db_path[256]; + FlattenPath(db_path, flatten_db_path, 256); + return log_dir + "/" + flatten_db_path + "_LOG"; } // Return the name of the old info log file for "dbname". -std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts) { +std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, + const std::string& db_path, const std::string& log_dir) { char buf[50]; snprintf(buf, sizeof(buf), "%llu", static_cast(ts)); - return dbname + "/LOG.old." + buf; + + if (log_dir.empty()) + return dbname + "/LOG.old." + buf; + + char flatten_db_path[256]; + FlattenPath(db_path, flatten_db_path, 256); + return log_dir + "/" + flatten_db_path + "_LOG.old." + buf; } diff --git a/db/filename.h b/db/filename.h index e6d6b19b5..ddf41cb4d 100644 --- a/db/filename.h +++ b/db/filename.h @@ -57,10 +57,12 @@ extern std::string LockFileName(const std::string& dbname); extern std::string TempFileName(const std::string& dbname, uint64_t number); // Return the name of the info log file for "dbname". -extern std::string InfoLogFileName(const std::string& dbname); +extern std::string InfoLogFileName(const std::string& dbname, + const std::string& db_path="", const std::string& log_dir=""); // Return the name of the old info log file for "dbname". -extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts); +extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts, + const std::string& db_path="", const std::string& log_dir=""); // If filename is a leveldb file, store the type of the file in *type. // The number encoded in the filename is stored in *number. If the diff --git a/include/leveldb/options.h b/include/leveldb/options.h index 1cf076e52..8644c7ba8 100644 --- a/include/leveldb/options.h +++ b/include/leveldb/options.h @@ -6,6 +6,7 @@ #define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ #include +#include namespace leveldb { @@ -212,6 +213,13 @@ struct Options { // Default value is 1800 (half an hour). int db_stats_log_interval; + // This specifies the log dir. + // If it is empty, the log files will be in the same dir as data. + // If it is non empty, the log files will be in the specified dir, + // and the db data dir's absolute path will be used as the log file + // name's prefix. + std::string db_log_dir; + // Create an Options object with default values for all fields. Options(); diff --git a/port/port_posix.h b/port/port_posix.h index 4ccd48fe7..b68910a03 100644 --- a/port/port_posix.h +++ b/port/port_posix.h @@ -320,7 +320,7 @@ inline bool BZip2_Compress(const char* input, size_t length, return true; return output; #endif - return NULL; + return false; } inline char* BZip2_Uncompress(const char* input_data, size_t input_length, diff --git a/util/options.cc b/util/options.cc index 51eb6c85f..8fb8000d0 100644 --- a/util/options.cc +++ b/util/options.cc @@ -38,7 +38,8 @@ Options::Options() statistics(NULL), disableDataSync(false), use_fsync(false), - db_stats_log_interval(1800) { + db_stats_log_interval(1800), + db_log_dir("") { } void @@ -84,6 +85,8 @@ Options::Dump( expanded_compaction_factor); Log(log," Options.max_grandparent_overlap_factor: %d", max_grandparent_overlap_factor); + Log(log," Options.db_log_dir: %s", + db_log_dir.c_str()); } // Options::Dump