put log in a seperate dir

Summary: added a new option db_log_dir, which points the log dir. Inside that dir, in order to make log names unique, the log file name is prefixed with the leveldb data dir absolute path.

Test Plan: db_test

Reviewers: dhruba

Reviewed By: dhruba

Differential Revision: https://reviews.facebook.net/D5205
main
heyongqiang 12 years ago
parent afb5f2210f
commit 0f43aa474e
  1. 20
      db/c.cc
  2. 30
      db/db_impl.cc
  3. 1
      db/db_impl.h
  4. 4
      db/db_stats_logger.cc
  5. 12
      db/db_test.cc
  6. 49
      db/filename.cc
  7. 6
      db/filename.h
  8. 8
      include/leveldb/options.h
  9. 2
      port/port_posix.h
  10. 5
      util/options.cc

@ -502,6 +502,26 @@ void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
opt->rep.compression = static_cast<CompressionType>(t);
}
void leveldb_options_set_disable_data_sync(
leveldb_options_t* opt, bool disable_data_sync) {
opt->rep.disableDataSync = disable_data_sync;
}
void leveldb_options_set_use_fsync(
leveldb_options_t* opt, bool use_fsync) {
opt->rep.use_fsync = use_fsync;
}
void leveldb_options_set_db_stats_log_interval(
leveldb_options_t* opt, int db_stats_log_interval) {
opt->rep.db_stats_log_interval = db_stats_log_interval;
}
void leveldb_options_set_db_log_dir(
leveldb_options_t* opt, const char* db_log_dir) {
opt->rep.db_log_dir = db_log_dir;
}
leveldb_comparator_t* leveldb_comparator_create(
void* state,
void (*destructor)(void*),

@ -98,12 +98,16 @@ Options SanitizeOptions(const std::string& dbname,
ClipToRange(&result.max_open_files, 20, 50000);
ClipToRange(&result.write_buffer_size, 64<<10, 1<<30);
ClipToRange(&result.block_size, 1<<10, 4<<20);
std::string db_absolute_path;
src.env->GetAbsolutePath(dbname, &db_absolute_path);
if (result.info_log == NULL) {
// Open a log file in the same directory as the db
src.env->CreateDir(dbname); // In case it does not exist
src.env->RenameFile(InfoLogFileName(dbname),
OldInfoLogFileName(dbname, src.env->NowMicros()));
Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log);
src.env->RenameFile(InfoLogFileName(dbname, db_absolute_path,
result.db_log_dir), OldInfoLogFileName(dbname,src.env->NowMicros(),
db_absolute_path, result.db_log_dir));
Status s = src.env->NewLogger(InfoLogFileName(dbname, db_absolute_path,
result.db_log_dir), &result.info_log);
if (!s.ok()) {
// No place suitable for logging
result.info_log = NULL;
@ -140,6 +144,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
mem_->Ref();
has_imm_.Release_Store(NULL);
env_->GetAbsolutePath(dbname, &db_absolute_path_);
stats_ = new CompactionStats[options.num_levels];
// Reserve ten files or so for other uses and give the rest to TableCache.
const int table_cache_size = options_.max_open_files - 10;
@ -247,7 +252,7 @@ void DBImpl::DeleteObsoleteFiles() {
env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose
uint64_t number;
FileType type;
std::vector<uint64_t> old_log_files_ts;
std::vector<std::string> old_log_files;
for (size_t i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type)) {
bool keep = true;
@ -272,7 +277,7 @@ void DBImpl::DeleteObsoleteFiles() {
case kInfoLogFile:
keep = true;
if (number != 0) {
old_log_files_ts.push_back(number);
old_log_files.push_back(filenames[i]);
}
break;
case kCurrentFile:
@ -299,15 +304,14 @@ void DBImpl::DeleteObsoleteFiles() {
}
// Delete old log files.
int old_log_file_count = old_log_files_ts.size();
if (old_log_file_count >= KEEP_LOG_FILE_NUM) {
std::sort(old_log_files_ts.begin(), old_log_files_ts.end());
int old_log_file_count = old_log_files.size();
if (old_log_file_count >= KEEP_LOG_FILE_NUM &&
!options_.db_log_dir.empty()) {
std::sort(old_log_files.begin(), old_log_files.end());
for (int i = 0; i >= (old_log_file_count - KEEP_LOG_FILE_NUM); i++) {
uint64_t ts = old_log_files_ts.at(i);
std::string to_delete = OldInfoLogFileName(dbname_, ts);
Log(options_.info_log, "Delete type=%d #%lld\n",
int(kInfoLogFile),
static_cast<unsigned long long>(ts));
std::string& to_delete = old_log_files.at(i);
Log(options_.info_log, "Delete type=%d %s\n",
int(kInfoLogFile), to_delete.c_str());
env_->DeleteFile(dbname_ + "/" + to_delete);
}
}

@ -210,6 +210,7 @@ class DBImpl : public DB {
CompactionStats* stats_;
static const int KEEP_LOG_FILE_NUM = 1000;
std::string db_absolute_path_;
// No copying allowed
DBImpl(const DBImpl&);

@ -74,11 +74,9 @@ void DBImpl::LogDBDeployStats() {
int64_t unix_ts;
env_->GetCurrentTime(&unix_ts);
std::string data_dir;
env_->GetAbsolutePath(dbname_, &data_dir);
logger_->Log_Deploy_Stats(version_info, host_name_,
data_dir, file_total_size, file_total_num, file_num_per_level,
db_absolute_path_, file_total_size, file_total_num, file_num_per_level,
data_size_per_level, unix_ts);
mutex_.Lock();

@ -154,6 +154,7 @@ class DBTest {
kFilter,
kUncompressed,
kNumLevel_3,
kDBLogDir,
kEnd
};
int option_config_;
@ -201,11 +202,14 @@ class DBTest {
options.filter_policy = filter_policy_;
break;
case kUncompressed:
options.compression = kNoCompression;
break;
options.compression = kNoCompression;
break;
case kNumLevel_3:
options.num_levels = 3;
break;
options.num_levels = 3;
break;
case kDBLogDir:
options.db_log_dir = test::TmpDir();
break;
default:
break;
}

@ -8,9 +8,37 @@
#include "db/dbformat.h"
#include "leveldb/env.h"
#include "util/logging.h"
#include <iostream>
namespace leveldb {
// Given a path, flatten the path name by replacing all chars not in
// {[0-9,a-z,A-Z,-,_,.]} with _. And append '\0' at the end.
// Return the number of chars stored in dest not including the trailing '\0'.
static int FlattenPath(const std::string& path, char* dest, int len) {
int write_idx = 0;
int i = 0;
int src_len = path.size();
while (i < src_len && write_idx < len - 1) {
if ((path[i] >= 'a' && path[i] <= 'z') ||
(path[i] >= '0' && path[i] <= '9') ||
(path[i] >= 'A' && path[i] <= 'Z') ||
path[i] == '-' ||
path[i] == '.' ||
path[i] == '_'){
dest[write_idx++] = path[i];
} else {
if (i > 0)
dest[write_idx++] = '_';
}
i++;
}
dest[write_idx] = '\0';
return write_idx;
}
// A utility routine: write "data" to the named file and Sync() it.
extern Status WriteStringToFileSync(Env* env, const Slice& data,
const std::string& fname);
@ -55,15 +83,28 @@ std::string TempFileName(const std::string& dbname, uint64_t number) {
return MakeFileName(dbname, number, "dbtmp");
}
std::string InfoLogFileName(const std::string& dbname) {
return dbname + "/LOG";
std::string InfoLogFileName(const std::string& dbname,
const std::string& db_path, const std::string& log_dir) {
if (log_dir.empty())
return dbname + "/LOG";
char flatten_db_path[256];
FlattenPath(db_path, flatten_db_path, 256);
return log_dir + "/" + flatten_db_path + "_LOG";
}
// Return the name of the old info log file for "dbname".
std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts) {
std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
const std::string& db_path, const std::string& log_dir) {
char buf[50];
snprintf(buf, sizeof(buf), "%llu", static_cast<unsigned long long>(ts));
return dbname + "/LOG.old." + buf;
if (log_dir.empty())
return dbname + "/LOG.old." + buf;
char flatten_db_path[256];
FlattenPath(db_path, flatten_db_path, 256);
return log_dir + "/" + flatten_db_path + "_LOG.old." + buf;
}

@ -57,10 +57,12 @@ extern std::string LockFileName(const std::string& dbname);
extern std::string TempFileName(const std::string& dbname, uint64_t number);
// Return the name of the info log file for "dbname".
extern std::string InfoLogFileName(const std::string& dbname);
extern std::string InfoLogFileName(const std::string& dbname,
const std::string& db_path="", const std::string& log_dir="");
// Return the name of the old info log file for "dbname".
extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts);
extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
const std::string& db_path="", const std::string& log_dir="");
// If filename is a leveldb file, store the type of the file in *type.
// The number encoded in the filename is stored in *number. If the

@ -6,6 +6,7 @@
#define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
#include <stddef.h>
#include <string>
namespace leveldb {
@ -212,6 +213,13 @@ struct Options {
// Default value is 1800 (half an hour).
int db_stats_log_interval;
// This specifies the log dir.
// If it is empty, the log files will be in the same dir as data.
// If it is non empty, the log files will be in the specified dir,
// and the db data dir's absolute path will be used as the log file
// name's prefix.
std::string db_log_dir;
// Create an Options object with default values for all fields.
Options();

@ -320,7 +320,7 @@ inline bool BZip2_Compress(const char* input, size_t length,
return true;
return output;
#endif
return NULL;
return false;
}
inline char* BZip2_Uncompress(const char* input_data, size_t input_length,

@ -38,7 +38,8 @@ Options::Options()
statistics(NULL),
disableDataSync(false),
use_fsync(false),
db_stats_log_interval(1800) {
db_stats_log_interval(1800),
db_log_dir("") {
}
void
@ -84,6 +85,8 @@ Options::Dump(
expanded_compaction_factor);
Log(log," Options.max_grandparent_overlap_factor: %d",
max_grandparent_overlap_factor);
Log(log," Options.db_log_dir: %s",
db_log_dir.c_str());
} // Options::Dump

Loading…
Cancel
Save