From c83b085770c003a2e2d30ceb24976aa388347d7f Mon Sep 17 00:00:00 2001
From: Lei Jin <lei@fb.com>
Date: Thu, 12 Jun 2014 10:06:18 -0700
Subject: [PATCH] prefetch bloom filter data block for L0 files

Summary: as title

Test Plan:
db_bench
the initial result is very promising. I will post results of complete
runs

Reviewers: dhruba, haobo, sdong, igor

Reviewed By: sdong

Subscribers: leveldb

Differential Revision: https://reviews.facebook.net/D18867
---
 db/version_set.cc           | 11 +++++++++++
 db/version_set.h            |  1 +
 port/port_posix.h           |  2 ++
 table/plain_table_reader.cc |  7 +++++++
 table/plain_table_reader.h  |  2 ++
 table/table_reader.h        |  3 +++
 util/dynamic_bloom.h        | 15 ++++++++++++---
 7 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/db/version_set.cc b/db/version_set.cc
index c6a9e6ab1..aee19dd1d 100644
--- a/db/version_set.cc
+++ b/db/version_set.cc
@@ -567,6 +567,16 @@ void Version::Get(const ReadOptions& options,
       continue;
     }
 
+    // Prefetch table data to avoid cache miss if possible
+    if (level == 0) {
+      for (int i = 0; i < num_files; ++i) {
+        auto* r = files_[0][i]->table_reader;
+        if (r) {
+          r->Prepare(ikey);
+        }
+      }
+    }
+
     // Get the list of files to search in this level
     FileMetaData* const* files = &files_[level][0];
 
@@ -607,6 +617,7 @@ void Version::Get(const ReadOptions& options,
 
     for (int32_t i = start_index; i < num_files;) {
       FileMetaData* f = files[i];
+
       // Check if key is within a file's range. If search left bound and right
       // bound point to the same find, we are sure key falls in range.
       assert(level == 0 || i == start_index ||
diff --git a/db/version_set.h b/db/version_set.h
index 7c8d7146e..446dcf133 100644
--- a/db/version_set.h
+++ b/db/version_set.h
@@ -92,6 +92,7 @@ class Version {
     FileMetaData* seek_file;
     int seek_file_level;
   };
+
   void Get(const ReadOptions&, const LookupKey& key, std::string* val,
            Status* status, MergeContext* merge_context, GetStats* stats,
            bool* value_found = nullptr);
diff --git a/port/port_posix.h b/port/port_posix.h
index d20a5dff2..f048d54ce 100644
--- a/port/port_posix.h
+++ b/port/port_posix.h
@@ -482,6 +482,8 @@ inline bool LZ4HC_Compress(const CompressionOptions &opts, const char* input,
 
 #define CACHE_LINE_SIZE 64U
 
+#define PREFETCH(addr, rw, locality) __builtin_prefetch(addr, rw, locality)
+
 } // namespace port
 } // namespace rocksdb
 
diff --git a/table/plain_table_reader.cc b/table/plain_table_reader.cc
index 139c30c30..e0da7e8c6 100644
--- a/table/plain_table_reader.cc
+++ b/table/plain_table_reader.cc
@@ -610,6 +610,13 @@ Status PlainTableReader::Next(uint32_t* offset, ParsedInternalKey* key,
   return Status::OK();
 }
 
+void PlainTableReader::Prepare(const Slice& target) {
+  if (enable_bloom_) {
+    uint32_t prefix_hash = GetSliceHash(GetPrefix(target));
+    bloom_.Prefetch(prefix_hash);
+  }
+}
+
 Status PlainTableReader::Get(const ReadOptions& ro, const Slice& target,
                              void* arg,
                              bool (*saver)(void*, const ParsedInternalKey&,
diff --git a/table/plain_table_reader.h b/table/plain_table_reader.h
index e20a109c7..9936847e3 100644
--- a/table/plain_table_reader.h
+++ b/table/plain_table_reader.h
@@ -57,6 +57,8 @@ class PlainTableReader: public TableReader {
 
   Iterator* NewIterator(const ReadOptions&, Arena* arena = nullptr) override;
 
+  void Prepare(const Slice& target);
+
   Status Get(const ReadOptions&, const Slice& key, void* arg,
              bool (*result_handler)(void* arg, const ParsedInternalKey& k,
                                     const Slice& v, bool),
diff --git a/table/table_reader.h b/table/table_reader.h
index 9238b880c..dec0cac3f 100644
--- a/table/table_reader.h
+++ b/table/table_reader.h
@@ -49,6 +49,9 @@ class TableReader {
 
   virtual std::shared_ptr<const TableProperties> GetTableProperties() const = 0;
 
+  // Prepare work that can be done before the real Get()
+  virtual void Prepare(const Slice& target) {}
+
   // Calls (*result_handler)(handle_context, ...) repeatedly, starting with
   // the entry found after a call to Seek(key), until result_handler returns
   // false, where k is the actual internal key for a row found and v as the
diff --git a/util/dynamic_bloom.h b/util/dynamic_bloom.h
index 2a0dc8434..4c4f7e1f9 100644
--- a/util/dynamic_bloom.h
+++ b/util/dynamic_bloom.h
@@ -5,12 +5,12 @@
 
 #pragma once
 
+#include <util/arena.h>
+#include <port/port_posix.h>
+
 #include <atomic>
 #include <memory>
 
-#include "port/port.h"
-#include <util/arena.h>
-
 namespace rocksdb {
 
 class Slice;
@@ -53,6 +53,8 @@ class DynamicBloom {
   // Multithreaded access to this function is OK
   bool MayContainHash(uint32_t hash) const;
 
+  void Prefetch(uint32_t h);
+
  private:
   uint32_t kTotalBits;
   uint32_t kNumBlocks;
@@ -71,6 +73,13 @@ inline bool DynamicBloom::MayContain(const Slice& key) const {
   return (MayContainHash(hash_func_(key)));
 }
 
+inline void DynamicBloom::Prefetch(uint32_t h) {
+  if (kNumBlocks != 0) {
+    uint32_t b = ((h >> 11 | (h << 21)) % kNumBlocks) * (CACHE_LINE_SIZE * 8);
+    PREFETCH(&(data_[b]), 0, 3);
+  }
+}
+
 inline bool DynamicBloom::MayContainHash(uint32_t h) const {
   assert(kNumBlocks > 0 || kTotalBits > 0);
   const uint32_t delta = (h >> 17) | (h << 15);  // Rotate right 17 bits