diff --git a/db/compaction.cc b/db/compaction.cc index 2802044a4..533fe497e 100644 --- a/db/compaction.cc +++ b/db/compaction.cc @@ -86,7 +86,8 @@ Compaction::~Compaction() { void Compaction::GenerateFileLevels() { input_levels_.resize(num_input_levels()); for (int which = 0; which < num_input_levels(); which++) { - DoGenerateFileLevel(&input_levels_[which], inputs_[which].files, &arena_); + DoGenerateLevelFilesBrief( + &input_levels_[which], inputs_[which].files, &arena_); } } diff --git a/db/compaction.h b/db/compaction.h index 7c490946a..5183822e3 100644 --- a/db/compaction.h +++ b/db/compaction.h @@ -89,8 +89,8 @@ class Compaction { return &inputs_[compaction_input_level].files; } - // Returns the FileLevel of the specified compaction input level. - FileLevel* input_levels(int compaction_input_level) { + // Returns the LevelFilesBrief of the specified compaction input level. + LevelFilesBrief* input_levels(int compaction_input_level) { return &input_levels_[compaction_input_level]; } @@ -193,7 +193,7 @@ class Compaction { autovector inputs_; // A copy of inputs_, organized more closely in memory - autovector input_levels_; + autovector input_levels_; // State used to check for number of of overlapping grandparent files // (grandparent == "output_level_ + 1") diff --git a/db/version_edit.h b/db/version_edit.h index ef883297a..fbe7e02d1 100644 --- a/db/version_edit.h +++ b/db/version_edit.h @@ -120,10 +120,10 @@ struct FdWithKeyRange { // Data structure to store an array of FdWithKeyRange in one level // Actual data is guaranteed to be stored closely -struct FileLevel { +struct LevelFilesBrief { size_t num_files; FdWithKeyRange* files; - FileLevel() { + LevelFilesBrief() { num_files = 0; files = nullptr; } diff --git a/db/version_set.cc b/db/version_set.cc index b121424d7..ec4df2823 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -46,10 +46,10 @@ namespace rocksdb { namespace { -// Find File in FileLevel data structure +// Find File in LevelFilesBrief data structure // Within an index range defined by left and right int FindFileInRange(const InternalKeyComparator& icmp, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice& key, uint32_t left, uint32_t right) { @@ -102,7 +102,7 @@ class FilePicker { std::vector* files, const Slice& user_key, const Slice& ikey, - autovector* file_levels, + autovector* file_levels, unsigned int num_levels, FileIndexer* file_indexer, const Comparator* user_comparator, @@ -114,7 +114,7 @@ class FilePicker { #ifndef NDEBUG files_(files), #endif - file_levels_(file_levels), + level_files_brief_(file_levels), user_key_(user_key), ikey_(ikey), file_indexer_(file_indexer), @@ -124,8 +124,8 @@ class FilePicker { search_ended_ = !PrepareNextLevel(); if (!search_ended_) { // Prefetch Level 0 table data to avoid cache miss if possible. - for (unsigned int i = 0; i < (*file_levels_)[0].num_files; ++i) { - auto* r = (*file_levels_)[0].files[i].fd.table_reader; + for (unsigned int i = 0; i < (*level_files_brief_)[0].num_files; ++i) { + auto* r = (*level_files_brief_)[0].files[i].fd.table_reader; if (r) { r->Prepare(ikey); } @@ -225,9 +225,9 @@ class FilePicker { #ifndef NDEBUG std::vector* files_; #endif - autovector* file_levels_; + autovector* level_files_brief_; bool search_ended_; - FileLevel* curr_file_level_; + LevelFilesBrief* curr_file_level_; unsigned int curr_index_in_curr_level_; unsigned int start_index_in_curr_level_; Slice user_key_; @@ -244,7 +244,7 @@ class FilePicker { bool PrepareNextLevel() { curr_level_++; while (curr_level_ < num_levels_) { - curr_file_level_ = &(*file_levels_)[curr_level_]; + curr_file_level_ = &(*level_files_brief_)[curr_level_]; if (curr_file_level_->num_files == 0) { // When current level is empty, the search bound generated from upper // level must be [0, -1] or [0, FileIndexer::kLevelMaxIndex] if it is @@ -331,12 +331,12 @@ Version::~Version() { } int FindFile(const InternalKeyComparator& icmp, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice& key) { return FindFileInRange(icmp, file_level, key, 0, file_level.num_files); } -void DoGenerateFileLevel(FileLevel* file_level, +void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level, const std::vector& files, Arena* arena) { assert(file_level); @@ -383,7 +383,7 @@ static bool BeforeFile(const Comparator* ucmp, bool SomeFileOverlapsRange( const InternalKeyComparator& icmp, bool disjoint_sorted_files, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice* smallest_user_key, const Slice* largest_user_key) { const Comparator* ucmp = icmp.user_comparator(); @@ -425,7 +425,7 @@ bool SomeFileOverlapsRange( class Version::LevelFileNumIterator : public Iterator { public: LevelFileNumIterator(const InternalKeyComparator& icmp, - const FileLevel* flevel) + const LevelFilesBrief* flevel) : icmp_(icmp), flevel_(flevel), index_(flevel->num_files), @@ -468,7 +468,7 @@ class Version::LevelFileNumIterator : public Iterator { virtual Status status() const { return Status::OK(); } private: const InternalKeyComparator icmp_; - const FileLevel* flevel_; + const LevelFilesBrief* flevel_; uint32_t index_; mutable FileDescriptor current_value_; }; @@ -582,7 +582,7 @@ Status Version::GetPropertiesOfAllTables(TablePropertiesCollection* props) { size_t Version::GetMemoryUsageByTableReaders() { size_t total_usage = 0; - for (auto& file_level : file_levels_) { + for (auto& file_level : level_files_brief_) { for (size_t i = 0; i < file_level.num_files; i++) { total_usage += cfd_->table_cache()->GetMemoryUsageByTableReader( vset_->env_options_, cfd_->internal_comparator(), @@ -618,8 +618,8 @@ void Version::AddIterators(const ReadOptions& read_options, assert(finalized_); // Merge all level zero files together since they may overlap - for (size_t i = 0; i < file_levels_[0].num_files; i++) { - const auto& file = file_levels_[0].files[i]; + for (size_t i = 0; i < level_files_brief_[0].num_files; i++) { + const auto& file = level_files_brief_[0].files[i]; merge_iter_builder->AddIterator(cfd_->table_cache()->NewIterator( read_options, soptions, cfd_->internal_comparator(), file.fd, nullptr, false, merge_iter_builder->GetArena())); @@ -629,14 +629,14 @@ void Version::AddIterators(const ReadOptions& read_options, // walks through the non-overlapping files in the level, opening them // lazily. for (int level = 1; level < num_levels_; level++) { - if (file_levels_[level].num_files != 0) { + if (level_files_brief_[level].num_files != 0) { merge_iter_builder->AddIterator(NewTwoLevelIterator( new LevelFileIteratorState( cfd_->table_cache(), read_options, soptions, cfd_->internal_comparator(), false /* for_compaction */, cfd_->ioptions()->prefix_extractor != nullptr), new LevelFileNumIterator(cfd_->internal_comparator(), - &file_levels_[level]), merge_iter_builder->GetArena())); + &level_files_brief_[level]), merge_iter_builder->GetArena())); } } } @@ -706,8 +706,9 @@ void Version::Get(const ReadOptions& read_options, db_statistics_, status->ok() ? GetContext::kNotFound : GetContext::kMerge, user_key, value, value_found, merge_context); - FilePicker fp(files_, user_key, ikey, &file_levels_, num_non_empty_levels_, - &file_indexer_, user_comparator_, internal_comparator_); + FilePicker fp(files_, user_key, ikey, &level_files_brief_, + num_non_empty_levels_, &file_indexer_, user_comparator_, + internal_comparator_); FdWithKeyRange* f = fp.GetNextFile(); while (f != nullptr) { *status = table_cache_->Get(read_options, *internal_comparator_, f->fd, @@ -758,10 +759,11 @@ void Version::Get(const ReadOptions& read_options, } } -void Version::GenerateFileLevels() { - file_levels_.resize(num_non_empty_levels_); +void Version::GenerateLevelFilesBrief() { + level_files_brief_.resize(num_non_empty_levels_); for (int level = 0; level < num_non_empty_levels_; level++) { - DoGenerateFileLevel(&file_levels_[level], files_[level], &arena_); + DoGenerateLevelFilesBrief( + &level_files_brief_[level], files_[level], &arena_); } } @@ -772,7 +774,7 @@ void Version::PrepareApply(const MutableCFOptions& mutable_cf_options, UpdateFilesBySize(); UpdateNumNonEmptyLevels(); file_indexer_.UpdateIndex(&arena_, num_non_empty_levels_, files_); - GenerateFileLevels(); + GenerateLevelFilesBrief(); } bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) { @@ -1046,7 +1048,7 @@ bool Version::OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key) { return SomeFileOverlapsRange(cfd_->internal_comparator(), (level > 0), - file_levels_[level], smallest_user_key, + level_files_brief_[level], smallest_user_key, largest_user_key); } @@ -1109,8 +1111,8 @@ void Version::GetOverlappingInputs(int level, hint_index, file_index); return; } - for (size_t i = 0; i < file_levels_[level].num_files; ) { - FdWithKeyRange* f = &(file_levels_[level].files[i++]); + for (size_t i = 0; i < level_files_brief_[level].num_files; ) { + FdWithKeyRange* f = &(level_files_brief_[level].files[i++]); const Slice file_start = ExtractUserKey(f->smallest_key); const Slice file_limit = ExtractUserKey(f->largest_key); if (begin != nullptr && user_cmp->Compare(file_limit, user_begin) < 0) { @@ -1166,7 +1168,7 @@ void Version::GetOverlappingInputsBinarySearch( while (!foundOverlap && min <= max) { mid = (min + max)/2; - FdWithKeyRange* f = &(file_levels_[level].files[mid]); + FdWithKeyRange* f = &(level_files_brief_[level].files[mid]); const Slice file_start = ExtractUserKey(f->smallest_key); const Slice file_limit = ExtractUserKey(f->largest_key); if (user_cmp->Compare(file_limit, user_begin) < 0) { @@ -1194,7 +1196,7 @@ void Version::GetOverlappingInputsBinarySearch( // The midIndex specifies the index of at least one file that // overlaps the specified range. From that file, iterate backward // and forward to find all overlapping files. -// Use FileLevel in searching, make it faster +// Use LevelFilesBrief in searching, make it faster void Version::ExtendOverlappingInputs( int level, const Slice& user_begin, @@ -1203,11 +1205,11 @@ void Version::ExtendOverlappingInputs( unsigned int midIndex) { const Comparator* user_cmp = cfd_->internal_comparator().user_comparator(); - const FdWithKeyRange* files = file_levels_[level].files; + const FdWithKeyRange* files = level_files_brief_[level].files; #ifndef NDEBUG { // assert that the file at midIndex overlaps with the range - assert(midIndex < file_levels_[level].num_files); + assert(midIndex < level_files_brief_[level].num_files); const FdWithKeyRange* f = &files[midIndex]; const Slice fstart = ExtractUserKey(f->smallest_key); const Slice flimit = ExtractUserKey(f->largest_key); @@ -1234,7 +1236,8 @@ void Version::ExtendOverlappingInputs( } } // check forward from 'mid+1' to higher indices - for (unsigned int i = midIndex+1; i < file_levels_[level].num_files; i++) { + for (unsigned int i = midIndex+1; + i < level_files_brief_[level].num_files; i++) { const FdWithKeyRange* f = &files[i]; const Slice file_start = ExtractUserKey(f->smallest_key); if (user_cmp->Compare(file_start, user_end) <= 0) { @@ -1268,8 +1271,8 @@ bool Version::HasOverlappingUserKey( } const Comparator* user_cmp = cfd_->internal_comparator().user_comparator(); - const FileLevel& file_level = file_levels_[level]; - const FdWithKeyRange* files = file_levels_[level].files; + const LevelFilesBrief& file_level = level_files_brief_[level]; + const FdWithKeyRange* files = level_files_brief_[level].files; const size_t kNumFiles = file_level.num_files; // Check the last file in inputs against the file after it @@ -2799,7 +2802,7 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) { for (int which = 0; which < c->num_input_levels(); which++) { if (c->input_levels(which)->num_files != 0) { if (c->level(which) == 0) { - const FileLevel* flevel = c->input_levels(which); + const LevelFilesBrief* flevel = c->input_levels(which); for (size_t i = 0; i < flevel->num_files; i++) { list[num++] = cfd->table_cache()->NewIterator( read_options, env_options_compactions_, diff --git a/db/version_set.h b/db/version_set.h index 95e38fca4..9a0c6733b 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -58,7 +58,7 @@ class MergeIteratorBuilder; // REQUIRES: "file_level.files" contains a sorted list of // non-overlapping files. extern int FindFile(const InternalKeyComparator& icmp, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice& key); // Returns true iff some file in "files" overlaps the user key range @@ -70,14 +70,14 @@ extern int FindFile(const InternalKeyComparator& icmp, extern bool SomeFileOverlapsRange( const InternalKeyComparator& icmp, bool disjoint_sorted_files, - const FileLevel& file_level, + const LevelFilesBrief& file_level, const Slice* smallest_user_key, const Slice* largest_user_key); -// Generate FileLevel from vector +// Generate LevelFilesBrief from vector // Would copy smallest_key and largest_key data to sequential memory // arena: Arena used to allocate the memory -extern void DoGenerateFileLevel(FileLevel* file_level, +extern void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level, const std::vector& files, Arena* arena); @@ -105,8 +105,8 @@ class Version { const MutableCFOptions& mutable_cf_options, std::vector& size_being_compacted); - // Generate file_levels_ from files_ - void GenerateFileLevels(); + // Generate level_files_brief_ from files_ + void GenerateLevelFilesBrief(); // Update scores, pre-calculated variables. It needs to be called before // applying the version to the version set. @@ -183,6 +183,12 @@ class Version { int NumberLevels() const { return num_levels_; } + // REQUIRES: This version has been saved (see VersionSet::SaveTo) + int NumNonEmptyLevels() const { + assert(finalized_); + return num_non_empty_levels_; + } + // REQUIRES: This version has been saved (see VersionSet::SaveTo) int NumLevelFiles(int level) const { assert(finalized_); @@ -263,6 +269,10 @@ class Version { return files_by_size_[level]; } + const LevelFilesBrief& GetLevelFilesBrief(int level) const { + return level_files_brief_[level]; + } + // REQUIRES: lock is held // Set the index that is used to offset into files_by_size_ to find // the next compaction candidate file. @@ -285,7 +295,6 @@ class Version { private: friend class VersionSet; friend class DBImpl; - friend class CompactedDBImpl; friend class ColumnFamilyData; friend class ForwardIterator; friend class InternalStats; @@ -321,7 +330,8 @@ class Version { TableCache* table_cache_; const MergeOperator* merge_operator_; - autovector file_levels_; // A copy of list of files per level + // A short brief metadata of files per level + autovector level_files_brief_; Logger* info_log_; Statistics* db_statistics_; int num_levels_; // Number of levels @@ -329,7 +339,7 @@ class Version { // is guaranteed to be empty. FileIndexer file_indexer_; VersionSet* vset_; // VersionSet to which this Version belongs - Arena arena_; // Used to allocate space for file_levels_ + Arena arena_; // Used to allocate space for level_files_brief_ Version* next_; // Next version in linked list Version* prev_; // Previous version in linked list int refs_; // Number of live refs to this version diff --git a/db/version_set_test.cc b/db/version_set_test.cc index 402762efa..9920a9e05 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -14,15 +14,15 @@ namespace rocksdb { -class GenerateFileLevelTest { +class GenerateLevelFilesBriefTest { public: std::vector files_; - FileLevel file_level_; + LevelFilesBrief file_level_; Arena arena_; - GenerateFileLevelTest() { } + GenerateLevelFilesBriefTest() { } - ~GenerateFileLevelTest() { + ~GenerateLevelFilesBriefTest() { for (unsigned int i = 0; i < files_.size(); i++) { delete files_[i]; } @@ -49,33 +49,33 @@ class GenerateFileLevelTest { } }; -TEST(GenerateFileLevelTest, Empty) { - DoGenerateFileLevel(&file_level_, files_, &arena_); +TEST(GenerateLevelFilesBriefTest, Empty) { + DoGenerateLevelFilesBrief(&file_level_, files_, &arena_); ASSERT_EQ(0u, file_level_.num_files); ASSERT_EQ(0, Compare()); } -TEST(GenerateFileLevelTest, Single) { +TEST(GenerateLevelFilesBriefTest, Single) { Add("p", "q"); - DoGenerateFileLevel(&file_level_, files_, &arena_); + DoGenerateLevelFilesBrief(&file_level_, files_, &arena_); ASSERT_EQ(1u, file_level_.num_files); ASSERT_EQ(0, Compare()); } -TEST(GenerateFileLevelTest, Multiple) { +TEST(GenerateLevelFilesBriefTest, Multiple) { Add("150", "200"); Add("200", "250"); Add("300", "350"); Add("400", "450"); - DoGenerateFileLevel(&file_level_, files_, &arena_); + DoGenerateLevelFilesBrief(&file_level_, files_, &arena_); ASSERT_EQ(4u, file_level_.num_files); ASSERT_EQ(0, Compare()); } class FindLevelFileTest { public: - FileLevel file_level_; + LevelFilesBrief file_level_; bool disjoint_sorted_files_; Arena arena_; diff --git a/utilities/compacted_db/compacted_db_impl.cc b/utilities/compacted_db/compacted_db_impl.cc index 775033e2a..a253153ae 100644 --- a/utilities/compacted_db/compacted_db_impl.cc +++ b/utilities/compacted_db/compacted_db_impl.cc @@ -104,27 +104,28 @@ Status CompactedDBImpl::Init(const Options& options) { } version_ = cfd_->GetSuperVersion()->current; user_comparator_ = cfd_->user_comparator(); + const LevelFilesBrief& l0 = version_->GetLevelFilesBrief(0); // L0 should not have files - if (version_->file_levels_[0].num_files > 1) { + if (l0.num_files > 1) { return Status::NotSupported("L0 contain more than 1 file"); } - if (version_->file_levels_[0].num_files == 1) { - if (version_->num_non_empty_levels_ > 1) { + if (l0.num_files == 1) { + if (version_->NumNonEmptyLevels() > 1) { return Status::NotSupported("Both L0 and other level contain files"); } - files_ = version_->file_levels_[0]; + files_ = l0; return Status::OK(); } - for (int i = 1; i < version_->num_non_empty_levels_ - 1; ++i) { - if (version_->file_levels_[i].num_files > 0) { + for (int i = 1; i < version_->NumNonEmptyLevels() - 1; ++i) { + if (version_->GetLevelFilesBrief(i).num_files > 0) { return Status::NotSupported("Other levels also contain files"); } } - int level = version_->num_non_empty_levels_ - 1; - if (version_->file_levels_[level].num_files > 0) { - files_ = version_->file_levels_[version_->num_non_empty_levels_ - 1]; + int level = version_->NumNonEmptyLevels() - 1; + if (version_->GetLevelFilesBrief(level).num_files > 0) { + files_ = version_->GetLevelFilesBrief(level); return Status::OK(); } return Status::NotSupported("no file exists"); diff --git a/utilities/compacted_db/compacted_db_impl.h b/utilities/compacted_db/compacted_db_impl.h index 49aca53b1..e1ac92dc4 100644 --- a/utilities/compacted_db/compacted_db_impl.h +++ b/utilities/compacted_db/compacted_db_impl.h @@ -86,7 +86,7 @@ class CompactedDBImpl : public DBImpl { ColumnFamilyData* cfd_; Version* version_; const Comparator* user_comparator_; - FileLevel files_; + LevelFilesBrief files_; // No copying allowed CompactedDBImpl(const CompactedDBImpl&);