From 0ab0242f373608c304aead5a291335a784ae0176 Mon Sep 17 00:00:00 2001
From: sdong <siying.d@fb.com>
Date: Tue, 9 Dec 2014 20:33:43 -0800
Subject: [PATCH] VersionBuilder to use unordered set and map to store added
 and deleted files

Summary: Set operations in VerisonBuilder is shown as a performance bottleneck of restarting DB when there are lots of files. Make both of added_files and deleted_files use unordered set or map. Only when adding the files, sort the added files.

Test Plan: make all check

Reviewers: yhchiang, rven, igor

Reviewed By: igor

Subscribers: hermanlee4, leveldb, dhruba, ljin

Differential Revision: https://reviews.facebook.net/D30051
---
 db/version_builder.cc | 70 ++++++++++++++++++++-----------------------
 1 file changed, 32 insertions(+), 38 deletions(-)

diff --git a/db/version_builder.cc b/db/version_builder.cc
index a360ab02a..abb0d4b58 100644
--- a/db/version_builder.cc
+++ b/db/version_builder.cc
@@ -16,6 +16,8 @@
 #include <inttypes.h>
 #include <algorithm>
 #include <set>
+#include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 #include "db/dbformat.h"
@@ -69,10 +71,10 @@ class VersionBuilder::Rep {
     }
   };
 
-  typedef std::set<FileMetaData*, FileComparator> FileSet;
   struct LevelState {
-    std::set<uint64_t> deleted_files;
-    FileSet* added_files;
+    std::unordered_set<uint64_t> deleted_files;
+    // Map from file number to file meta data.
+    std::unordered_map<uint64_t, FileMetaData*> added_files;
   };
 
   const EnvOptions& env_options_;
@@ -93,25 +95,13 @@ class VersionBuilder::Rep {
     level_nonzero_cmp_.sort_method = FileComparator::kLevelNon0;
     level_nonzero_cmp_.internal_comparator =
         base_vstorage_->InternalComparator();
-
-    levels_[0].added_files = new FileSet(level_zero_cmp_);
-    for (int level = 1; level < base_vstorage_->num_levels(); level++) {
-      levels_[level].added_files = new FileSet(level_nonzero_cmp_);
-    }
   }
 
   ~Rep() {
     for (int level = 0; level < base_vstorage_->num_levels(); level++) {
-      const FileSet* added = levels_[level].added_files;
-      std::vector<FileMetaData*> to_unref;
-      to_unref.reserve(added->size());
-      for (FileSet::const_iterator it = added->begin(); it != added->end();
-           ++it) {
-        to_unref.push_back(*it);
-      }
-      delete added;
-      for (uint32_t i = 0; i < to_unref.size(); i++) {
-        FileMetaData* f = to_unref[i];
+      const auto& added = levels_[level].added_files;
+      for (auto& pair : added) {
+        FileMetaData* f = pair.second;
         f->refs--;
         if (f->refs <= 0) {
           if (f->table_reader_handle) {
@@ -175,27 +165,20 @@ class VersionBuilder::Rep {
     // is possibly moved from lower level to higher level in current
     // version
     for (int l = level + 1; !found && l < base_vstorage_->num_levels(); l++) {
-      const FileSet* added = levels_[l].added_files;
-      for (FileSet::const_iterator added_iter = added->begin();
-           added_iter != added->end(); ++added_iter) {
-        FileMetaData* f = *added_iter;
-        if (f->fd.GetNumber() == number) {
-          found = true;
-          break;
-        }
+      auto& level_added = levels_[l].added_files;
+      auto got = level_added.find(number);
+      if (got != level_added.end()) {
+        found = true;
+        break;
       }
     }
 
     // maybe this file was added in a previous edit that was Applied
     if (!found) {
-      const FileSet* added = levels_[level].added_files;
-      for (FileSet::const_iterator added_iter = added->begin();
-           added_iter != added->end(); ++added_iter) {
-        FileMetaData* f = *added_iter;
-        if (f->fd.GetNumber() == number) {
-          found = true;
-          break;
-        }
+      auto& level_added = levels_[level].added_files;
+      auto got = level_added.find(number);
+      if (got != level_added.end()) {
+        found = true;
       }
     }
     if (!found) {
@@ -224,8 +207,10 @@ class VersionBuilder::Rep {
       FileMetaData* f = new FileMetaData(new_file.second);
       f->refs = 1;
 
+      assert(levels_[level].added_files.find(f->fd.GetNumber()) ==
+             levels_[level].added_files.end());
       levels_[level].deleted_files.erase(f->fd.GetNumber());
-      levels_[level].added_files->insert(f);
+      levels_[level].added_files[f->fd.GetNumber()] = f;
     }
   }
 
@@ -241,8 +226,16 @@ class VersionBuilder::Rep {
       const auto& base_files = base_vstorage_->LevelFiles(level);
       auto base_iter = base_files.begin();
       auto base_end = base_files.end();
-      const auto& added_files = *levels_[level].added_files;
-      vstorage->Reserve(level, base_files.size() + added_files.size());
+      const auto& unordered_added_files = levels_[level].added_files;
+      vstorage->Reserve(level,
+                        base_files.size() + unordered_added_files.size());
+
+      // Sort added files for the level.
+      autovector<FileMetaData*> added_files;
+      for (const auto& pair : unordered_added_files) {
+        added_files.push_back(pair.second);
+      }
+      std::sort(added_files.begin(), added_files.end(), cmp);
 
       for (const auto& added : added_files) {
         // Add all smaller files listed in base_
@@ -266,7 +259,8 @@ class VersionBuilder::Rep {
   void LoadTableHandlers() {
     assert(table_cache_ != nullptr);
     for (int level = 0; level < base_vstorage_->num_levels(); level++) {
-      for (auto& file_meta : *(levels_[level].added_files)) {
+      for (auto& file_meta_pair : levels_[level].added_files) {
+        auto* file_meta = file_meta_pair.second;
         assert(!file_meta->table_reader_handle);
         table_cache_->FindTable(
             env_options_, *(base_vstorage_->InternalComparator()),