From 7b9f1349591ef3107e738fd9a98539d0e16306d0 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Tue, 4 Feb 2014 17:45:19 -0800 Subject: [PATCH] [CF] Move InternalStats to ColumnFamilyData Summary: InternalStats is a messy thing, keeping both DB data and column family data. However, it's better off living in ColumnFamilyData than in DBImpl. For now, at least. Test Plan: make check Reviewers: dhruba, kailiu, haobo, sdong CC: leveldb Differential Revision: https://reviews.facebook.net/D15879 --- db/column_family.cc | 29 +++++++++++++++++++---------- db/column_family.h | 11 ++++++++--- db/db_impl.cc | 24 ++++++++++++------------ db/db_impl.h | 2 -- db/version_set.cc | 2 +- 5 files changed, 40 insertions(+), 28 deletions(-) diff --git a/db/column_family.cc b/db/column_family.cc index ac85692e4..b029bff21 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -14,6 +14,7 @@ #include #include "db/version_set.h" +#include "db/internal_stats.h" #include "db/compaction_picker.h" #include "db/table_properties_collector.h" #include "util/hash_skiplist_rep.h" @@ -130,7 +131,7 @@ void SuperVersion::Init(MemTable* new_mem, MemTableListVersion* new_imm, ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name, Version* dummy_versions, const ColumnFamilyOptions& options, - Logger* logger) + const Options* db_options) : id_(id), name_(name), dummy_versions_(dummy_versions), @@ -147,12 +148,16 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name, prev_(nullptr), log_number_(0), need_slowdown_for_num_level0_files_(false) { - if (options_.compaction_style == kCompactionStyleUniversal) { - compaction_picker_.reset(new UniversalCompactionPicker( - &options_, &internal_comparator_, logger)); - } else { - compaction_picker_.reset( - new LevelCompactionPicker(&options_, &internal_comparator_, logger)); + if (db_options != nullptr) { + internal_stats_.reset(new InternalStats(options.num_levels, db_options->env, + db_options->statistics.get())); + if (options_.compaction_style == kCompactionStyleUniversal) { + compaction_picker_.reset(new UniversalCompactionPicker( + &options_, &internal_comparator_, db_options->info_log.get())); + } else { + compaction_picker_.reset(new LevelCompactionPicker( + &options_, &internal_comparator_, db_options->info_log.get())); + } } } @@ -180,6 +185,10 @@ ColumnFamilyData::~ColumnFamilyData() { } } +InternalStats* ColumnFamilyData::internal_stats() { + return internal_stats_.get(); +} + void ColumnFamilyData::SetCurrent(Version* current) { current_ = current; need_slowdown_for_num_level0_files_ = @@ -221,11 +230,11 @@ SuperVersion* ColumnFamilyData::InstallSuperVersion( return nullptr; } -ColumnFamilySet::ColumnFamilySet(Logger* logger) +ColumnFamilySet::ColumnFamilySet(const Options* db_options) : max_column_family_(0), dummy_cfd_(new ColumnFamilyData(0, "", nullptr, ColumnFamilyOptions(), nullptr)), - logger_(logger) { + db_options_(db_options) { // initialize linked list dummy_cfd_->prev_.store(dummy_cfd_); dummy_cfd_->next_.store(dummy_cfd_); @@ -281,7 +290,7 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily( assert(column_families_.find(name) == column_families_.end()); column_families_.insert({name, id}); ColumnFamilyData* new_cfd = - new ColumnFamilyData(id, name, dummy_versions, options, logger_); + new ColumnFamilyData(id, name, dummy_versions, options, db_options_); column_family_data_.insert({id, new_cfd}); max_column_family_ = std::max(max_column_family_, id); // add to linked list diff --git a/db/column_family.h b/db/column_family.h index 390cef839..f912c3386 100644 --- a/db/column_family.h +++ b/db/column_family.h @@ -27,6 +27,7 @@ class MemTableListVersion; class CompactionPicker; class Compaction; class InternalKey; +class InternalStats; // holds references to memtable, all immutable memtables and version struct SuperVersion { @@ -65,7 +66,7 @@ class ColumnFamilyData { public: ColumnFamilyData(uint32_t id, const std::string& name, Version* dummy_versions, const ColumnFamilyOptions& options, - Logger* logger); + const Options* db_options); ~ColumnFamilyData(); uint32_t GetID() const { return id_; } @@ -77,6 +78,7 @@ class ColumnFamilyData { uint64_t GetLogNumber() const { return log_number_; } ColumnFamilyOptions* options() { return &options_; } + InternalStats* internal_stats(); MemTableList* imm() { return &imm_; } MemTable* mem() { return mem_; } @@ -133,6 +135,8 @@ class ColumnFamilyData { ColumnFamilyOptions options_; + std::unique_ptr internal_stats_; + MemTable* mem_; MemTableList imm_; SuperVersion* super_version_; @@ -182,7 +186,7 @@ class ColumnFamilySet { ColumnFamilyData* current_; }; - explicit ColumnFamilySet(Logger* logger); + explicit ColumnFamilySet(const Options* db_options_); ~ColumnFamilySet(); ColumnFamilyData* GetDefault() const; @@ -215,7 +219,8 @@ class ColumnFamilySet { std::vector droppped_column_families_; uint32_t max_column_family_; ColumnFamilyData* dummy_cfd_; - Logger* logger_; + // TODO(icanadi) change to DBOptions + const Options* const db_options_; }; class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables { diff --git a/db/db_impl.cc b/db/db_impl.cc index 073f1c4b8..02e31cb58 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -227,8 +227,6 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname) last_stats_dump_time_microsec_(0), default_interval_to_delete_obsolete_WAL_(600), flush_on_destroy_(false), - internal_stats_(options.num_levels, options.env, - options.statistics.get()), delayed_writes_(0), storage_options_(options), bg_work_gate_closed_(false), @@ -1044,7 +1042,7 @@ Status DBImpl::WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem, stats.micros = env_->NowMicros() - start_micros; stats.bytes_written = meta.file_size; stats.files_out_levelnp1 = 1; - internal_stats_.AddCompactionStats(level, stats); + cfd->internal_stats()->AddCompactionStats(level, stats); RecordTick(options_.statistics.get(), COMPACT_WRITE_BYTES, meta.file_size); return s; } @@ -1132,7 +1130,7 @@ Status DBImpl::WriteLevel0Table(ColumnFamilyData* cfd, InternalStats::CompactionStats stats; stats.micros = env_->NowMicros() - start_micros; stats.bytes_written = meta.file_size; - internal_stats_.AddCompactionStats(level, stats); + cfd->internal_stats()->AddCompactionStats(level, stats); RecordTick(options_.statistics.get(), COMPACT_WRITE_BYTES, meta.file_size); return s; } @@ -2608,8 +2606,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact, LogFlush(options_.info_log); mutex_.Lock(); - internal_stats_.AddCompactionStats(compact->compaction->output_level(), - stats); + cfd->internal_stats()->AddCompactionStats(compact->compaction->output_level(), + stats); // if there were any unused file number (mostly in case of // compaction error), free up the entry from pending_putputs @@ -3349,7 +3347,8 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) { delayed = sw.ElapsedMicros(); } RecordTick(options_.statistics.get(), STALL_L0_SLOWDOWN_MICROS, delayed); - internal_stats_.RecordWriteStall(InternalStats::LEVEL0_SLOWDOWN, delayed); + cfd->internal_stats()->RecordWriteStall(InternalStats::LEVEL0_SLOWDOWN, + delayed); allow_delay = false; // Do not delay a single write more than once mutex_.Lock(); delayed_writes_++; @@ -3375,8 +3374,8 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) { } RecordTick(options_.statistics.get(), STALL_MEMTABLE_COMPACTION_MICROS, stall); - internal_stats_.RecordWriteStall(InternalStats::MEMTABLE_COMPACTION, - stall); + cfd->internal_stats()->RecordWriteStall( + InternalStats::MEMTABLE_COMPACTION, stall); } else if (cfd->current()->NumLevelFiles(0) >= cfd->options()->level0_stop_writes_trigger) { // There are too many level-0 files. @@ -3390,7 +3389,8 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) { stall = sw.ElapsedMicros(); } RecordTick(options_.statistics.get(), STALL_L0_NUM_FILES_MICROS, stall); - internal_stats_.RecordWriteStall(InternalStats::LEVEL0_NUM_FILES, stall); + cfd->internal_stats()->RecordWriteStall(InternalStats::LEVEL0_NUM_FILES, + stall); } else if (allow_hard_rate_limit_delay && options_.hard_rate_limit > 1.0 && (score = cfd->current()->MaxCompactionScore()) > cfd->options()->hard_rate_limit) { @@ -3404,7 +3404,7 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) { env_->SleepForMicroseconds(1000); delayed = sw.ElapsedMicros(); } - internal_stats_.RecordLevelNSlowdown(max_level, delayed); + cfd->internal_stats()->RecordLevelNSlowdown(max_level, delayed); // Make sure the following value doesn't round to zero. uint64_t rate_limit = std::max((delayed / 1000), (uint64_t) 1); rate_limit_delay_millis += rate_limit; @@ -3505,7 +3505,7 @@ bool DBImpl::GetProperty(const ColumnFamilyHandle& column_family, MutexLock l(&mutex_); auto cfd = versions_->GetColumnFamilySet()->GetColumnFamily(column_family.id); assert(cfd != nullptr); - return internal_stats_.GetProperty(property, value, cfd); + return cfd->internal_stats()->GetProperty(property, value, cfd); } void DBImpl::GetApproximateSizes(const ColumnFamilyHandle& column_family, diff --git a/db/db_impl.h b/db/db_impl.h index 0e001e837..c52264fa6 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -460,8 +460,6 @@ class DBImpl : public DB { bool flush_on_destroy_; // Used when disableWAL is true. - InternalStats internal_stats_; - static const int KEEP_LOG_FILE_NUM = 1000; std::string db_absolute_path_; diff --git a/db/version_set.cc b/db/version_set.cc index 7fbe44047..95bdf656b 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -1374,7 +1374,7 @@ class VersionSet::Builder { VersionSet::VersionSet(const std::string& dbname, const Options* options, const EnvOptions& storage_options, TableCache* table_cache) - : column_family_set_(new ColumnFamilySet(options->info_log.get())), + : column_family_set_(new ColumnFamilySet(options)), env_(options->env), dbname_(dbname), options_(options),