From 834c67d77f81e892ddf03d942e9d1a2e38a9787b Mon Sep 17 00:00:00 2001 From: Lei Jin Date: Tue, 28 Oct 2014 10:03:13 -0700 Subject: [PATCH] rename FileLevel to LevelFilesBrief / unfriend CompactedDBImpl Summary: We have several different types of data structures for file information. FileLevel is kinda of confusing since it only contains file range and fd. Rename it to LevelFilesBrief to make it clear. Unfriend CompactedDBImpl as a by product Test Plan: make release / make all will run full test with all stacked diffs Reviewers: sdong, yhchiang, rven, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D27585 --- db/compaction.cc | 3 +- db/compaction.h | 6 +- db/version_edit.h | 4 +- db/version_set.cc | 75 +++++++++++---------- db/version_set.h | 28 +++++--- db/version_set_test.cc | 22 +++--- utilities/compacted_db/compacted_db_impl.cc | 19 +++--- utilities/compacted_db/compacted_db_impl.h | 2 +- 8 files changed, 87 insertions(+), 72 deletions(-) diff --git a/db/compaction.cc b/db/compaction.cc index 2802044a4..533fe497e 100644 --- a/db/compaction.cc +++ b/db/compaction.cc @@ -86,7 +86,8 @@ Compaction::~Compaction() { void Compaction::GenerateFileLevels() { input_levels_.resize(num_input_levels()); for (int which = 0; which < num_input_levels(); which++) { - DoGenerateFileLevel(&input_levels_[which], inputs_[which].files, &arena_); + DoGenerateLevelFilesBrief( + &input_levels_[which], inputs_[which].files, &arena_); } } diff --git a/db/compaction.h b/db/compaction.h index 7c490946a..5183822e3 100644 --- a/db/compaction.h +++ b/db/compaction.h @@ -89,8 +89,8 @@ class Compaction { return &inputs_[compaction_input_level].files; } - // Returns the FileLevel of the specified compaction input level. - FileLevel* input_levels(int compaction_input_level) { + // Returns the LevelFilesBrief of the specified compaction input level. + LevelFilesBrief* input_levels(int compaction_input_level) { return &input_levels_[compaction_input_level]; } @@ -193,7 +193,7 @@ class Compaction { autovector inputs_; // A copy of inputs_, organized more closely in memory - autovector input_levels_; + autovector input_levels_; // State used to check for number of of overlapping grandparent files // (grandparent == "output_level_ + 1") diff --git a/db/version_edit.h b/db/version_edit.h index ef883297a..fbe7e02d1 100644 --- a/db/version_edit.h +++ b/db/version_edit.h @@ -120,10 +120,10 @@ struct FdWithKeyRange { // Data structure to store an array of FdWithKeyRange in one level // Actual data is guaranteed to be stored closely -struct FileLevel { +struct LevelFilesBrief { size_t num_files; FdWithKeyRange* files; - FileLevel() { + LevelFilesBrief() { num_files = 0; files = nullptr; } diff --git a/db/version_set.cc b/db/version_set.cc index b121424d7..ec4df2823 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -46,10 +46,10 @@ namespace rocksdb { namespace { -// Find File in FileLevel data structure +// Find File in LevelFilesBrief data structure // Within an index range defined by left and right int FindFileInRange(const InternalKeyComparator& icmp, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice& key, uint32_t left, uint32_t right) { @@ -102,7 +102,7 @@ class FilePicker { std::vector* files, const Slice& user_key, const Slice& ikey, - autovector* file_levels, + autovector* file_levels, unsigned int num_levels, FileIndexer* file_indexer, const Comparator* user_comparator, @@ -114,7 +114,7 @@ class FilePicker { #ifndef NDEBUG files_(files), #endif - file_levels_(file_levels), + level_files_brief_(file_levels), user_key_(user_key), ikey_(ikey), file_indexer_(file_indexer), @@ -124,8 +124,8 @@ class FilePicker { search_ended_ = !PrepareNextLevel(); if (!search_ended_) { // Prefetch Level 0 table data to avoid cache miss if possible. - for (unsigned int i = 0; i < (*file_levels_)[0].num_files; ++i) { - auto* r = (*file_levels_)[0].files[i].fd.table_reader; + for (unsigned int i = 0; i < (*level_files_brief_)[0].num_files; ++i) { + auto* r = (*level_files_brief_)[0].files[i].fd.table_reader; if (r) { r->Prepare(ikey); } @@ -225,9 +225,9 @@ class FilePicker { #ifndef NDEBUG std::vector* files_; #endif - autovector* file_levels_; + autovector* level_files_brief_; bool search_ended_; - FileLevel* curr_file_level_; + LevelFilesBrief* curr_file_level_; unsigned int curr_index_in_curr_level_; unsigned int start_index_in_curr_level_; Slice user_key_; @@ -244,7 +244,7 @@ class FilePicker { bool PrepareNextLevel() { curr_level_++; while (curr_level_ < num_levels_) { - curr_file_level_ = &(*file_levels_)[curr_level_]; + curr_file_level_ = &(*level_files_brief_)[curr_level_]; if (curr_file_level_->num_files == 0) { // When current level is empty, the search bound generated from upper // level must be [0, -1] or [0, FileIndexer::kLevelMaxIndex] if it is @@ -331,12 +331,12 @@ Version::~Version() { } int FindFile(const InternalKeyComparator& icmp, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice& key) { return FindFileInRange(icmp, file_level, key, 0, file_level.num_files); } -void DoGenerateFileLevel(FileLevel* file_level, +void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level, const std::vector& files, Arena* arena) { assert(file_level); @@ -383,7 +383,7 @@ static bool BeforeFile(const Comparator* ucmp, bool SomeFileOverlapsRange( const InternalKeyComparator& icmp, bool disjoint_sorted_files, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice* smallest_user_key, const Slice* largest_user_key) { const Comparator* ucmp = icmp.user_comparator(); @@ -425,7 +425,7 @@ bool SomeFileOverlapsRange( class Version::LevelFileNumIterator : public Iterator { public: LevelFileNumIterator(const InternalKeyComparator& icmp, - const FileLevel* flevel) + const LevelFilesBrief* flevel) : icmp_(icmp), flevel_(flevel), index_(flevel->num_files), @@ -468,7 +468,7 @@ class Version::LevelFileNumIterator : public Iterator { virtual Status status() const { return Status::OK(); } private: const InternalKeyComparator icmp_; - const FileLevel* flevel_; + const LevelFilesBrief* flevel_; uint32_t index_; mutable FileDescriptor current_value_; }; @@ -582,7 +582,7 @@ Status Version::GetPropertiesOfAllTables(TablePropertiesCollection* props) { size_t Version::GetMemoryUsageByTableReaders() { size_t total_usage = 0; - for (auto& file_level : file_levels_) { + for (auto& file_level : level_files_brief_) { for (size_t i = 0; i < file_level.num_files; i++) { total_usage += cfd_->table_cache()->GetMemoryUsageByTableReader( vset_->env_options_, cfd_->internal_comparator(), @@ -618,8 +618,8 @@ void Version::AddIterators(const ReadOptions& read_options, assert(finalized_); // Merge all level zero files together since they may overlap - for (size_t i = 0; i < file_levels_[0].num_files; i++) { - const auto& file = file_levels_[0].files[i]; + for (size_t i = 0; i < level_files_brief_[0].num_files; i++) { + const auto& file = level_files_brief_[0].files[i]; merge_iter_builder->AddIterator(cfd_->table_cache()->NewIterator( read_options, soptions, cfd_->internal_comparator(), file.fd, nullptr, false, merge_iter_builder->GetArena())); @@ -629,14 +629,14 @@ void Version::AddIterators(const ReadOptions& read_options, // walks through the non-overlapping files in the level, opening them // lazily. for (int level = 1; level < num_levels_; level++) { - if (file_levels_[level].num_files != 0) { + if (level_files_brief_[level].num_files != 0) { merge_iter_builder->AddIterator(NewTwoLevelIterator( new LevelFileIteratorState( cfd_->table_cache(), read_options, soptions, cfd_->internal_comparator(), false /* for_compaction */, cfd_->ioptions()->prefix_extractor != nullptr), new LevelFileNumIterator(cfd_->internal_comparator(), - &file_levels_[level]), merge_iter_builder->GetArena())); + &level_files_brief_[level]), merge_iter_builder->GetArena())); } } } @@ -706,8 +706,9 @@ void Version::Get(const ReadOptions& read_options, db_statistics_, status->ok() ? GetContext::kNotFound : GetContext::kMerge, user_key, value, value_found, merge_context); - FilePicker fp(files_, user_key, ikey, &file_levels_, num_non_empty_levels_, - &file_indexer_, user_comparator_, internal_comparator_); + FilePicker fp(files_, user_key, ikey, &level_files_brief_, + num_non_empty_levels_, &file_indexer_, user_comparator_, + internal_comparator_); FdWithKeyRange* f = fp.GetNextFile(); while (f != nullptr) { *status = table_cache_->Get(read_options, *internal_comparator_, f->fd, @@ -758,10 +759,11 @@ void Version::Get(const ReadOptions& read_options, } } -void Version::GenerateFileLevels() { - file_levels_.resize(num_non_empty_levels_); +void Version::GenerateLevelFilesBrief() { + level_files_brief_.resize(num_non_empty_levels_); for (int level = 0; level < num_non_empty_levels_; level++) { - DoGenerateFileLevel(&file_levels_[level], files_[level], &arena_); + DoGenerateLevelFilesBrief( + &level_files_brief_[level], files_[level], &arena_); } } @@ -772,7 +774,7 @@ void Version::PrepareApply(const MutableCFOptions& mutable_cf_options, UpdateFilesBySize(); UpdateNumNonEmptyLevels(); file_indexer_.UpdateIndex(&arena_, num_non_empty_levels_, files_); - GenerateFileLevels(); + GenerateLevelFilesBrief(); } bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) { @@ -1046,7 +1048,7 @@ bool Version::OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key) { return SomeFileOverlapsRange(cfd_->internal_comparator(), (level > 0), - file_levels_[level], smallest_user_key, + level_files_brief_[level], smallest_user_key, largest_user_key); } @@ -1109,8 +1111,8 @@ void Version::GetOverlappingInputs(int level, hint_index, file_index); return; } - for (size_t i = 0; i < file_levels_[level].num_files; ) { - FdWithKeyRange* f = &(file_levels_[level].files[i++]); + for (size_t i = 0; i < level_files_brief_[level].num_files; ) { + FdWithKeyRange* f = &(level_files_brief_[level].files[i++]); const Slice file_start = ExtractUserKey(f->smallest_key); const Slice file_limit = ExtractUserKey(f->largest_key); if (begin != nullptr && user_cmp->Compare(file_limit, user_begin) < 0) { @@ -1166,7 +1168,7 @@ void Version::GetOverlappingInputsBinarySearch( while (!foundOverlap && min <= max) { mid = (min + max)/2; - FdWithKeyRange* f = &(file_levels_[level].files[mid]); + FdWithKeyRange* f = &(level_files_brief_[level].files[mid]); const Slice file_start = ExtractUserKey(f->smallest_key); const Slice file_limit = ExtractUserKey(f->largest_key); if (user_cmp->Compare(file_limit, user_begin) < 0) { @@ -1194,7 +1196,7 @@ void Version::GetOverlappingInputsBinarySearch( // The midIndex specifies the index of at least one file that // overlaps the specified range. From that file, iterate backward // and forward to find all overlapping files. -// Use FileLevel in searching, make it faster +// Use LevelFilesBrief in searching, make it faster void Version::ExtendOverlappingInputs( int level, const Slice& user_begin, @@ -1203,11 +1205,11 @@ void Version::ExtendOverlappingInputs( unsigned int midIndex) { const Comparator* user_cmp = cfd_->internal_comparator().user_comparator(); - const FdWithKeyRange* files = file_levels_[level].files; + const FdWithKeyRange* files = level_files_brief_[level].files; #ifndef NDEBUG { // assert that the file at midIndex overlaps with the range - assert(midIndex < file_levels_[level].num_files); + assert(midIndex < level_files_brief_[level].num_files); const FdWithKeyRange* f = &files[midIndex]; const Slice fstart = ExtractUserKey(f->smallest_key); const Slice flimit = ExtractUserKey(f->largest_key); @@ -1234,7 +1236,8 @@ void Version::ExtendOverlappingInputs( } } // check forward from 'mid+1' to higher indices - for (unsigned int i = midIndex+1; i < file_levels_[level].num_files; i++) { + for (unsigned int i = midIndex+1; + i < level_files_brief_[level].num_files; i++) { const FdWithKeyRange* f = &files[i]; const Slice file_start = ExtractUserKey(f->smallest_key); if (user_cmp->Compare(file_start, user_end) <= 0) { @@ -1268,8 +1271,8 @@ bool Version::HasOverlappingUserKey( } const Comparator* user_cmp = cfd_->internal_comparator().user_comparator(); - const FileLevel& file_level = file_levels_[level]; - const FdWithKeyRange* files = file_levels_[level].files; + const LevelFilesBrief& file_level = level_files_brief_[level]; + const FdWithKeyRange* files = level_files_brief_[level].files; const size_t kNumFiles = file_level.num_files; // Check the last file in inputs against the file after it @@ -2799,7 +2802,7 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) { for (int which = 0; which < c->num_input_levels(); which++) { if (c->input_levels(which)->num_files != 0) { if (c->level(which) == 0) { - const FileLevel* flevel = c->input_levels(which); + const LevelFilesBrief* flevel = c->input_levels(which); for (size_t i = 0; i < flevel->num_files; i++) { list[num++] = cfd->table_cache()->NewIterator( read_options, env_options_compactions_, diff --git a/db/version_set.h b/db/version_set.h index 95e38fca4..9a0c6733b 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -58,7 +58,7 @@ class MergeIteratorBuilder; // REQUIRES: "file_level.files" contains a sorted list of // non-overlapping files. extern int FindFile(const InternalKeyComparator& icmp, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice& key); // Returns true iff some file in "files" overlaps the user key range @@ -70,14 +70,14 @@ extern int FindFile(const InternalKeyComparator& icmp, extern bool SomeFileOverlapsRange( const InternalKeyComparator& icmp, bool disjoint_sorted_files, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice* smallest_user_key, const Slice* largest_user_key); -// Generate FileLevel from vector +// Generate LevelFilesBrief from vector // Would copy smallest_key and largest_key data to sequential memory // arena: Arena used to allocate the memory -extern void DoGenerateFileLevel(FileLevel* file_level, +extern void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level, const std::vector& files, Arena* arena); @@ -105,8 +105,8 @@ class Version { const MutableCFOptions& mutable_cf_options, std::vector& size_being_compacted); - // Generate file_levels_ from files_ - void GenerateFileLevels(); + // Generate level_files_brief_ from files_ + void GenerateLevelFilesBrief(); // Update scores, pre-calculated variables. It needs to be called before // applying the version to the version set. @@ -183,6 +183,12 @@ class Version { int NumberLevels() const { return num_levels_; } + // REQUIRES: This version has been saved (see VersionSet::SaveTo) + int NumNonEmptyLevels() const { + assert(finalized_); + return num_non_empty_levels_; + } + // REQUIRES: This version has been saved (see VersionSet::SaveTo) int NumLevelFiles(int level) const { assert(finalized_); @@ -263,6 +269,10 @@ class Version { return files_by_size_[level]; } + const LevelFilesBrief& GetLevelFilesBrief(int level) const { + return level_files_brief_[level]; + } + // REQUIRES: lock is held // Set the index that is used to offset into files_by_size_ to find // the next compaction candidate file. @@ -285,7 +295,6 @@ class Version { private: friend class VersionSet; friend class DBImpl; - friend class CompactedDBImpl; friend class ColumnFamilyData; friend class ForwardIterator; friend class InternalStats; @@ -321,7 +330,8 @@ class Version { TableCache* table_cache_; const MergeOperator* merge_operator_; - autovector file_levels_; // A copy of list of files per level + // A short brief metadata of files per level + autovector level_files_brief_; Logger* info_log_; Statistics* db_statistics_; int num_levels_; // Number of levels @@ -329,7 +339,7 @@ class Version { // is guaranteed to be empty. FileIndexer file_indexer_; VersionSet* vset_; // VersionSet to which this Version belongs - Arena arena_; // Used to allocate space for file_levels_ + Arena arena_; // Used to allocate space for level_files_brief_ Version* next_; // Next version in linked list Version* prev_; // Previous version in linked list int refs_; // Number of live refs to this version diff --git a/db/version_set_test.cc b/db/version_set_test.cc index 402762efa..9920a9e05 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -14,15 +14,15 @@ namespace rocksdb { -class GenerateFileLevelTest { +class GenerateLevelFilesBriefTest { public: std::vector files_; - FileLevel file_level_; + LevelFilesBrief file_level_; Arena arena_; - GenerateFileLevelTest() { } + GenerateLevelFilesBriefTest() { } - ~GenerateFileLevelTest() { + ~GenerateLevelFilesBriefTest() { for (unsigned int i = 0; i < files_.size(); i++) { delete files_[i]; } @@ -49,33 +49,33 @@ class GenerateFileLevelTest { } }; -TEST(GenerateFileLevelTest, Empty) { - DoGenerateFileLevel(&file_level_, files_, &arena_); +TEST(GenerateLevelFilesBriefTest, Empty) { + DoGenerateLevelFilesBrief(&file_level_, files_, &arena_); ASSERT_EQ(0u, file_level_.num_files); ASSERT_EQ(0, Compare()); } -TEST(GenerateFileLevelTest, Single) { +TEST(GenerateLevelFilesBriefTest, Single) { Add("p", "q"); - DoGenerateFileLevel(&file_level_, files_, &arena_); + DoGenerateLevelFilesBrief(&file_level_, files_, &arena_); ASSERT_EQ(1u, file_level_.num_files); ASSERT_EQ(0, Compare()); } -TEST(GenerateFileLevelTest, Multiple) { +TEST(GenerateLevelFilesBriefTest, Multiple) { Add("150", "200"); Add("200", "250"); Add("300", "350"); Add("400", "450"); - DoGenerateFileLevel(&file_level_, files_, &arena_); + DoGenerateLevelFilesBrief(&file_level_, files_, &arena_); ASSERT_EQ(4u, file_level_.num_files); ASSERT_EQ(0, Compare()); } class FindLevelFileTest { public: - FileLevel file_level_; + LevelFilesBrief file_level_; bool disjoint_sorted_files_; Arena arena_; diff --git a/utilities/compacted_db/compacted_db_impl.cc b/utilities/compacted_db/compacted_db_impl.cc index 775033e2a..a253153ae 100644 --- a/utilities/compacted_db/compacted_db_impl.cc +++ b/utilities/compacted_db/compacted_db_impl.cc @@ -104,27 +104,28 @@ Status CompactedDBImpl::Init(const Options& options) { } version_ = cfd_->GetSuperVersion()->current; user_comparator_ = cfd_->user_comparator(); + const LevelFilesBrief& l0 = version_->GetLevelFilesBrief(0); // L0 should not have files - if (version_->file_levels_[0].num_files > 1) { + if (l0.num_files > 1) { return Status::NotSupported("L0 contain more than 1 file"); } - if (version_->file_levels_[0].num_files == 1) { - if (version_->num_non_empty_levels_ > 1) { + if (l0.num_files == 1) { + if (version_->NumNonEmptyLevels() > 1) { return Status::NotSupported("Both L0 and other level contain files"); } - files_ = version_->file_levels_[0]; + files_ = l0; return Status::OK(); } - for (int i = 1; i < version_->num_non_empty_levels_ - 1; ++i) { - if (version_->file_levels_[i].num_files > 0) { + for (int i = 1; i < version_->NumNonEmptyLevels() - 1; ++i) { + if (version_->GetLevelFilesBrief(i).num_files > 0) { return Status::NotSupported("Other levels also contain files"); } } - int level = version_->num_non_empty_levels_ - 1; - if (version_->file_levels_[level].num_files > 0) { - files_ = version_->file_levels_[version_->num_non_empty_levels_ - 1]; + int level = version_->NumNonEmptyLevels() - 1; + if (version_->GetLevelFilesBrief(level).num_files > 0) { + files_ = version_->GetLevelFilesBrief(level); return Status::OK(); } return Status::NotSupported("no file exists"); diff --git a/utilities/compacted_db/compacted_db_impl.h b/utilities/compacted_db/compacted_db_impl.h index 49aca53b1..e1ac92dc4 100644 --- a/utilities/compacted_db/compacted_db_impl.h +++ b/utilities/compacted_db/compacted_db_impl.h @@ -86,7 +86,7 @@ class CompactedDBImpl : public DBImpl { ColumnFamilyData* cfd_; Version* version_; const Comparator* user_comparator_; - FileLevel files_; + LevelFilesBrief files_; // No copying allowed CompactedDBImpl(const CompactedDBImpl&);