rename FileLevel to LevelFilesBrief / unfriend CompactedDBImpl

Summary:
We have several different types of data structures for file information.
FileLevel is kinda of confusing since it only contains file range and
fd. Rename it to LevelFilesBrief to make it clear.
Unfriend CompactedDBImpl as a by product

Test Plan:
make release / make all
will run full test with all stacked diffs

Reviewers: sdong, yhchiang, rven, igor

Reviewed By: igor

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D27585

db/compaction.cc

@@ -86,7 +86,8 @@ Compaction::~Compaction() {
 void Compaction::GenerateFileLevels() {
   input_levels_.resize(num_input_levels());
   for (int which = 0; which < num_input_levels(); which++) {
-    DoGenerateFileLevel(&input_levels_[which], inputs_[which].files, &arena_);
+    DoGenerateLevelFilesBrief(
+        &input_levels_[which], inputs_[which].files, &arena_);
   }
 }
 

db/compaction.h

@@ -89,8 +89,8 @@ class Compaction {
     return &inputs_[compaction_input_level].files;
   }
 
-  // Returns the FileLevel of the specified compaction input level.
-  FileLevel* input_levels(int compaction_input_level) {
+  // Returns the LevelFilesBrief of the specified compaction input level.
+  LevelFilesBrief* input_levels(int compaction_input_level) {
     return &input_levels_[compaction_input_level];
   }
 
@@ -193,7 +193,7 @@ class Compaction {
   autovector<CompactionInputFiles> inputs_;
 
   // A copy of inputs_, organized more closely in memory
-  autovector<FileLevel, 2> input_levels_;
+  autovector<LevelFilesBrief, 2> input_levels_;
 
   // State used to check for number of of overlapping grandparent files
   // (grandparent == "output_level_ + 1")

db/version_edit.h

@@ -120,10 +120,10 @@ struct FdWithKeyRange {
 
 // Data structure to store an array of FdWithKeyRange in one level
 // Actual data is guaranteed to be stored closely
-struct FileLevel {
+struct LevelFilesBrief {
   size_t num_files;
   FdWithKeyRange* files;
-  FileLevel() {
+  LevelFilesBrief() {
     num_files = 0;
     files = nullptr;
   }

db/version_set.cc

@@ -46,10 +46,10 @@ namespace rocksdb {
 
 namespace {
 
-// Find File in FileLevel data structure
+// Find File in LevelFilesBrief data structure
 // Within an index range defined by left and right
 int FindFileInRange(const InternalKeyComparator& icmp,
-    const FileLevel& file_level,
+    const LevelFilesBrief& file_level,
     const Slice& key,
     uint32_t left,
     uint32_t right) {
@@ -102,7 +102,7 @@ class FilePicker {
       std::vector<FileMetaData*>* files,
       const Slice& user_key,
       const Slice& ikey,
-      autovector<FileLevel>* file_levels,
+      autovector<LevelFilesBrief>* file_levels,
       unsigned int num_levels,
       FileIndexer* file_indexer,
       const Comparator* user_comparator,
@@ -114,7 +114,7 @@ class FilePicker {
 #ifndef NDEBUG
         files_(files),
 #endif
-        file_levels_(file_levels),
+        level_files_brief_(file_levels),
         user_key_(user_key),
         ikey_(ikey),
         file_indexer_(file_indexer),
@@ -124,8 +124,8 @@ class FilePicker {
     search_ended_ = !PrepareNextLevel();
     if (!search_ended_) {
       // Prefetch Level 0 table data to avoid cache miss if possible.
-      for (unsigned int i = 0; i < (*file_levels_)[0].num_files; ++i) {
-        auto* r = (*file_levels_)[0].files[i].fd.table_reader;
+      for (unsigned int i = 0; i < (*level_files_brief_)[0].num_files; ++i) {
+        auto* r = (*level_files_brief_)[0].files[i].fd.table_reader;
         if (r) {
           r->Prepare(ikey);
         }
@@ -225,9 +225,9 @@ class FilePicker {
 #ifndef NDEBUG
   std::vector<FileMetaData*>* files_;
 #endif
-  autovector<FileLevel>* file_levels_;
+  autovector<LevelFilesBrief>* level_files_brief_;
   bool search_ended_;
-  FileLevel* curr_file_level_;
+  LevelFilesBrief* curr_file_level_;
   unsigned int curr_index_in_curr_level_;
   unsigned int start_index_in_curr_level_;
   Slice user_key_;
@@ -244,7 +244,7 @@ class FilePicker {
   bool PrepareNextLevel() {
     curr_level_++;
     while (curr_level_ < num_levels_) {
-      curr_file_level_ = &(*file_levels_)[curr_level_];
+      curr_file_level_ = &(*level_files_brief_)[curr_level_];
       if (curr_file_level_->num_files == 0) {
         // When current level is empty, the search bound generated from upper
         // level must be [0, -1] or [0, FileIndexer::kLevelMaxIndex] if it is
@@ -331,12 +331,12 @@ Version::~Version() {
 }
 
 int FindFile(const InternalKeyComparator& icmp,
-             const FileLevel& file_level,
+             const LevelFilesBrief& file_level,
              const Slice& key) {
   return FindFileInRange(icmp, file_level, key, 0, file_level.num_files);
 }
 
-void DoGenerateFileLevel(FileLevel* file_level,
+void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
         const std::vector<FileMetaData*>& files,
         Arena* arena) {
   assert(file_level);
@@ -383,7 +383,7 @@ static bool BeforeFile(const Comparator* ucmp,
 bool SomeFileOverlapsRange(
     const InternalKeyComparator& icmp,
     bool disjoint_sorted_files,
-    const FileLevel& file_level,
+    const LevelFilesBrief& file_level,
     const Slice* smallest_user_key,
     const Slice* largest_user_key) {
   const Comparator* ucmp = icmp.user_comparator();
@@ -425,7 +425,7 @@ bool SomeFileOverlapsRange(
 class Version::LevelFileNumIterator : public Iterator {
  public:
   LevelFileNumIterator(const InternalKeyComparator& icmp,
-                       const FileLevel* flevel)
+                       const LevelFilesBrief* flevel)
       : icmp_(icmp),
         flevel_(flevel),
         index_(flevel->num_files),
@@ -468,7 +468,7 @@ class Version::LevelFileNumIterator : public Iterator {
   virtual Status status() const { return Status::OK(); }
  private:
   const InternalKeyComparator icmp_;
-  const FileLevel* flevel_;
+  const LevelFilesBrief* flevel_;
   uint32_t index_;
   mutable FileDescriptor current_value_;
 };
@@ -582,7 +582,7 @@ Status Version::GetPropertiesOfAllTables(TablePropertiesCollection* props) {
 
 size_t Version::GetMemoryUsageByTableReaders() {
   size_t total_usage = 0;
-  for (auto& file_level : file_levels_) {
+  for (auto& file_level : level_files_brief_) {
     for (size_t i = 0; i < file_level.num_files; i++) {
       total_usage += cfd_->table_cache()->GetMemoryUsageByTableReader(
           vset_->env_options_, cfd_->internal_comparator(),
@@ -618,8 +618,8 @@ void Version::AddIterators(const ReadOptions& read_options,
   assert(finalized_);
 
   // Merge all level zero files together since they may overlap
-  for (size_t i = 0; i < file_levels_[0].num_files; i++) {
-    const auto& file = file_levels_[0].files[i];
+  for (size_t i = 0; i < level_files_brief_[0].num_files; i++) {
+    const auto& file = level_files_brief_[0].files[i];
     merge_iter_builder->AddIterator(cfd_->table_cache()->NewIterator(
         read_options, soptions, cfd_->internal_comparator(), file.fd, nullptr,
         false, merge_iter_builder->GetArena()));
@@ -629,14 +629,14 @@ void Version::AddIterators(const ReadOptions& read_options,
   // walks through the non-overlapping files in the level, opening them
   // lazily.
   for (int level = 1; level < num_levels_; level++) {
-    if (file_levels_[level].num_files != 0) {
+    if (level_files_brief_[level].num_files != 0) {
       merge_iter_builder->AddIterator(NewTwoLevelIterator(
           new LevelFileIteratorState(
               cfd_->table_cache(), read_options, soptions,
               cfd_->internal_comparator(), false /* for_compaction */,
               cfd_->ioptions()->prefix_extractor != nullptr),
           new LevelFileNumIterator(cfd_->internal_comparator(),
-              &file_levels_[level]), merge_iter_builder->GetArena()));
+              &level_files_brief_[level]), merge_iter_builder->GetArena()));
     }
   }
 }
@@ -706,8 +706,9 @@ void Version::Get(const ReadOptions& read_options,
       db_statistics_, status->ok() ? GetContext::kNotFound : GetContext::kMerge,
       user_key, value, value_found, merge_context);
 
-  FilePicker fp(files_, user_key, ikey, &file_levels_, num_non_empty_levels_,
-      &file_indexer_, user_comparator_, internal_comparator_);
+  FilePicker fp(files_, user_key, ikey, &level_files_brief_,
+      num_non_empty_levels_, &file_indexer_, user_comparator_,
+      internal_comparator_);
   FdWithKeyRange* f = fp.GetNextFile();
   while (f != nullptr) {
     *status = table_cache_->Get(read_options, *internal_comparator_, f->fd,
@@ -758,10 +759,11 @@ void Version::Get(const ReadOptions& read_options,
   }
 }
 
-void Version::GenerateFileLevels() {
-  file_levels_.resize(num_non_empty_levels_);
+void Version::GenerateLevelFilesBrief() {
+  level_files_brief_.resize(num_non_empty_levels_);
   for (int level = 0; level < num_non_empty_levels_; level++) {
-    DoGenerateFileLevel(&file_levels_[level], files_[level], &arena_);
+    DoGenerateLevelFilesBrief(
+        &level_files_brief_[level], files_[level], &arena_);
   }
 }
 
@@ -772,7 +774,7 @@ void Version::PrepareApply(const MutableCFOptions& mutable_cf_options,
   UpdateFilesBySize();
   UpdateNumNonEmptyLevels();
   file_indexer_.UpdateIndex(&arena_, num_non_empty_levels_, files_);
-  GenerateFileLevels();
+  GenerateLevelFilesBrief();
 }
 
 bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) {
@@ -1046,7 +1048,7 @@ bool Version::OverlapInLevel(int level,
                              const Slice* smallest_user_key,
                              const Slice* largest_user_key) {
   return SomeFileOverlapsRange(cfd_->internal_comparator(), (level > 0),
-                               file_levels_[level], smallest_user_key,
+                               level_files_brief_[level], smallest_user_key,
                                largest_user_key);
 }
 
@@ -1109,8 +1111,8 @@ void Version::GetOverlappingInputs(int level,
       hint_index, file_index);
     return;
   }
-  for (size_t i = 0; i < file_levels_[level].num_files; ) {
-    FdWithKeyRange* f = &(file_levels_[level].files[i++]);
+  for (size_t i = 0; i < level_files_brief_[level].num_files; ) {
+    FdWithKeyRange* f = &(level_files_brief_[level].files[i++]);
     const Slice file_start = ExtractUserKey(f->smallest_key);
     const Slice file_limit = ExtractUserKey(f->largest_key);
     if (begin != nullptr && user_cmp->Compare(file_limit, user_begin) < 0) {
@@ -1166,7 +1168,7 @@ void Version::GetOverlappingInputsBinarySearch(
 
   while (!foundOverlap && min <= max) {
     mid = (min + max)/2;
-    FdWithKeyRange* f = &(file_levels_[level].files[mid]);
+    FdWithKeyRange* f = &(level_files_brief_[level].files[mid]);
     const Slice file_start = ExtractUserKey(f->smallest_key);
     const Slice file_limit = ExtractUserKey(f->largest_key);
     if (user_cmp->Compare(file_limit, user_begin) < 0) {
@@ -1194,7 +1196,7 @@ void Version::GetOverlappingInputsBinarySearch(
 // The midIndex specifies the index of at least one file that
 // overlaps the specified range. From that file, iterate backward
 // and forward to find all overlapping files.
-// Use FileLevel in searching, make it faster
+// Use LevelFilesBrief in searching, make it faster
 void Version::ExtendOverlappingInputs(
     int level,
     const Slice& user_begin,
@@ -1203,11 +1205,11 @@ void Version::ExtendOverlappingInputs(
     unsigned int midIndex) {
 
   const Comparator* user_cmp = cfd_->internal_comparator().user_comparator();
-  const FdWithKeyRange* files = file_levels_[level].files;
+  const FdWithKeyRange* files = level_files_brief_[level].files;
 #ifndef NDEBUG
   {
     // assert that the file at midIndex overlaps with the range
-    assert(midIndex < file_levels_[level].num_files);
+    assert(midIndex < level_files_brief_[level].num_files);
     const FdWithKeyRange* f = &files[midIndex];
     const Slice fstart = ExtractUserKey(f->smallest_key);
     const Slice flimit = ExtractUserKey(f->largest_key);
@@ -1234,7 +1236,8 @@ void Version::ExtendOverlappingInputs(
     }
   }
   // check forward from 'mid+1' to higher indices
-  for (unsigned int i = midIndex+1; i < file_levels_[level].num_files; i++) {
+  for (unsigned int i = midIndex+1;
+       i < level_files_brief_[level].num_files; i++) {
     const FdWithKeyRange* f = &files[i];
     const Slice file_start = ExtractUserKey(f->smallest_key);
     if (user_cmp->Compare(file_start, user_end) <= 0) {
@@ -1268,8 +1271,8 @@ bool Version::HasOverlappingUserKey(
   }
 
   const Comparator* user_cmp = cfd_->internal_comparator().user_comparator();
-  const FileLevel& file_level = file_levels_[level];
-  const FdWithKeyRange* files = file_levels_[level].files;
+  const LevelFilesBrief& file_level = level_files_brief_[level];
+  const FdWithKeyRange* files = level_files_brief_[level].files;
   const size_t kNumFiles = file_level.num_files;
 
   // Check the last file in inputs against the file after it
@@ -2799,7 +2802,7 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
   for (int which = 0; which < c->num_input_levels(); which++) {
     if (c->input_levels(which)->num_files != 0) {
       if (c->level(which) == 0) {
-        const FileLevel* flevel = c->input_levels(which);
+        const LevelFilesBrief* flevel = c->input_levels(which);
         for (size_t i = 0; i < flevel->num_files; i++) {
           list[num++] = cfd->table_cache()->NewIterator(
               read_options, env_options_compactions_,

db/version_set.h

@@ -58,7 +58,7 @@ class MergeIteratorBuilder;
 // REQUIRES: "file_level.files" contains a sorted list of
 // non-overlapping files.
 extern int FindFile(const InternalKeyComparator& icmp,
-                    const FileLevel& file_level,
+                    const LevelFilesBrief& file_level,
                     const Slice& key);
 
 // Returns true iff some file in "files" overlaps the user key range
@@ -70,14 +70,14 @@ extern int FindFile(const InternalKeyComparator& icmp,
 extern bool SomeFileOverlapsRange(
     const InternalKeyComparator& icmp,
     bool disjoint_sorted_files,
-    const FileLevel& file_level,
+    const LevelFilesBrief& file_level,
     const Slice* smallest_user_key,
     const Slice* largest_user_key);
 
-// Generate FileLevel from vector<FdWithKeyRange*>
+// Generate LevelFilesBrief from vector<FdWithKeyRange*>
 // Would copy smallest_key and largest_key data to sequential memory
 // arena: Arena used to allocate the memory
-extern void DoGenerateFileLevel(FileLevel* file_level,
+extern void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
         const std::vector<FileMetaData*>& files,
         Arena* arena);
 
@@ -105,8 +105,8 @@ class Version {
       const MutableCFOptions& mutable_cf_options,
       std::vector<uint64_t>& size_being_compacted);
 
-  // Generate file_levels_ from files_
-  void GenerateFileLevels();
+  // Generate level_files_brief_ from files_
+  void GenerateLevelFilesBrief();
 
   // Update scores, pre-calculated variables. It needs to be called before
   // applying the version to the version set.
@@ -183,6 +183,12 @@ class Version {
 
   int NumberLevels() const { return num_levels_; }
 
+  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+  int NumNonEmptyLevels() const {
+    assert(finalized_);
+    return num_non_empty_levels_;
+  }
+
   // REQUIRES: This version has been saved (see VersionSet::SaveTo)
   int NumLevelFiles(int level) const {
     assert(finalized_);
@@ -263,6 +269,10 @@ class Version {
     return files_by_size_[level];
   }
 
+  const LevelFilesBrief& GetLevelFilesBrief(int level) const {
+    return level_files_brief_[level];
+  }
+
   // REQUIRES: lock is held
   // Set the index that is used to offset into files_by_size_ to find
   // the next compaction candidate file.
@@ -285,7 +295,6 @@ class Version {
  private:
   friend class VersionSet;
   friend class DBImpl;
-  friend class CompactedDBImpl;
   friend class ColumnFamilyData;
   friend class ForwardIterator;
   friend class InternalStats;
@@ -321,7 +330,8 @@ class Version {
   TableCache* table_cache_;
   const MergeOperator* merge_operator_;
 
-  autovector<FileLevel> file_levels_;   // A copy of list of files per level
+  // A short brief metadata of files per level
+  autovector<LevelFilesBrief> level_files_brief_;
   Logger* info_log_;
   Statistics* db_statistics_;
   int num_levels_;              // Number of levels
@@ -329,7 +339,7 @@ class Version {
                                 // is guaranteed to be empty.
   FileIndexer file_indexer_;
   VersionSet* vset_;            // VersionSet to which this Version belongs
-  Arena arena_;                 // Used to allocate space for file_levels_
+  Arena arena_;                 // Used to allocate space for level_files_brief_
   Version* next_;               // Next version in linked list
   Version* prev_;               // Previous version in linked list
   int refs_;                    // Number of live refs to this version

db/version_set_test.cc

@@ -14,15 +14,15 @@
 
 namespace rocksdb {
 
-class GenerateFileLevelTest {
+class GenerateLevelFilesBriefTest {
  public:
   std::vector<FileMetaData*> files_;
-  FileLevel file_level_;
+  LevelFilesBrief file_level_;
   Arena arena_;
 
-  GenerateFileLevelTest() { }
+  GenerateLevelFilesBriefTest() { }
 
-  ~GenerateFileLevelTest() {
+  ~GenerateLevelFilesBriefTest() {
     for (unsigned int i = 0; i < files_.size(); i++) {
       delete files_[i];
     }
@@ -49,33 +49,33 @@ class GenerateFileLevelTest {
   }
 };
 
-TEST(GenerateFileLevelTest, Empty) {
-  DoGenerateFileLevel(&file_level_, files_, &arena_);
+TEST(GenerateLevelFilesBriefTest, Empty) {
+  DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
   ASSERT_EQ(0u, file_level_.num_files);
   ASSERT_EQ(0, Compare());
 }
 
-TEST(GenerateFileLevelTest, Single) {
+TEST(GenerateLevelFilesBriefTest, Single) {
   Add("p", "q");
-  DoGenerateFileLevel(&file_level_, files_, &arena_);
+  DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
   ASSERT_EQ(1u, file_level_.num_files);
   ASSERT_EQ(0, Compare());
 }
 
 
-TEST(GenerateFileLevelTest, Multiple) {
+TEST(GenerateLevelFilesBriefTest, Multiple) {
   Add("150", "200");
   Add("200", "250");
   Add("300", "350");
   Add("400", "450");
-  DoGenerateFileLevel(&file_level_, files_, &arena_);
+  DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
   ASSERT_EQ(4u, file_level_.num_files);
   ASSERT_EQ(0, Compare());
 }
 
 class FindLevelFileTest {
  public:
-  FileLevel file_level_;
+  LevelFilesBrief file_level_;
   bool disjoint_sorted_files_;
   Arena arena_;
 

utilities/compacted_db/compacted_db_impl.cc

@@ -104,27 +104,28 @@ Status CompactedDBImpl::Init(const Options& options) {
   }
   version_ = cfd_->GetSuperVersion()->current;
   user_comparator_ = cfd_->user_comparator();
+  const LevelFilesBrief& l0 = version_->GetLevelFilesBrief(0);
   // L0 should not have files
-  if (version_->file_levels_[0].num_files > 1) {
+  if (l0.num_files > 1) {
     return Status::NotSupported("L0 contain more than 1 file");
   }
-  if (version_->file_levels_[0].num_files == 1) {
-    if (version_->num_non_empty_levels_ > 1) {
+  if (l0.num_files == 1) {
+    if (version_->NumNonEmptyLevels() > 1) {
       return Status::NotSupported("Both L0 and other level contain files");
     }
-    files_ = version_->file_levels_[0];
+    files_ = l0;
     return Status::OK();
   }
 
-  for (int i = 1; i < version_->num_non_empty_levels_ - 1; ++i) {
-    if (version_->file_levels_[i].num_files > 0) {
+  for (int i = 1; i < version_->NumNonEmptyLevels() - 1; ++i) {
+    if (version_->GetLevelFilesBrief(i).num_files > 0) {
       return Status::NotSupported("Other levels also contain files");
     }
   }
 
-  int level = version_->num_non_empty_levels_ - 1;
-  if (version_->file_levels_[level].num_files > 0) {
-    files_ = version_->file_levels_[version_->num_non_empty_levels_ - 1];
+  int level = version_->NumNonEmptyLevels() - 1;
+  if (version_->GetLevelFilesBrief(level).num_files > 0) {
+    files_ = version_->GetLevelFilesBrief(level);
     return Status::OK();
   }
   return Status::NotSupported("no file exists");

utilities/compacted_db/compacted_db_impl.h

@@ -86,7 +86,7 @@ class CompactedDBImpl : public DBImpl {
   ColumnFamilyData* cfd_;
   Version* version_;
   const Comparator* user_comparator_;
-  FileLevel files_;
+  LevelFilesBrief files_;
 
   // No copying allowed
   CompactedDBImpl(const CompactedDBImpl&);