From fd1da221115611cd8dbfc3898aa4616da1d41eca Mon Sep 17 00:00:00 2001
From: Yanqin Jin <yanqin@fb.com>
Date: Mon, 9 Mar 2020 18:42:27 -0700
Subject: [PATCH] Support options.max_open_files != -1 with FIFO compaction
 (#6503)

Summary:
Allow user to specify options.max_open_files != -1 with FIFO compaction.
If max_open_files != -1, not all table files are kept open.
In the past, FIFO style compaction requires all table files to be open in order
to read file creation time from table properties. Later, we added file creation
time to MANIFEST, making it possible to read file creation time without opening
file.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6503

Test Plan: make check

Differential Revision: D20353758

Pulled By: riversand963

fbshipit-source-id: ba5c61a648419e47e9ef6d74e0e280e3ee24f296
---
 db/compaction/compaction_picker_fifo.cc | 20 ++++++++------------
 db/db_compaction_test.cc                | 25 +++++++++++++++++++++++++
 include/rocksdb/advanced_options.h      |  2 --
 3 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/db/compaction/compaction_picker_fifo.cc b/db/compaction/compaction_picker_fifo.cc
index 030da619c..947c40cf6 100644
--- a/db/compaction/compaction_picker_fifo.cc
+++ b/db/compaction/compaction_picker_fifo.cc
@@ -70,18 +70,14 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
   // avoid underflow
   if (current_time > mutable_cf_options.ttl) {
     for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
-      auto f = *ritr;
-      if (f->fd.table_reader != nullptr &&
-          f->fd.table_reader->GetTableProperties() != nullptr) {
-        auto creation_time =
-            f->fd.table_reader->GetTableProperties()->creation_time;
-        if (creation_time == 0 ||
-            creation_time >= (current_time - mutable_cf_options.ttl)) {
-          break;
-        }
-        total_size -= f->compensated_file_size;
-        inputs[0].files.push_back(f);
+      FileMetaData* f = *ritr;
+      uint64_t creation_time = f->TryGetFileCreationTime();
+      if (creation_time == kUnknownFileCreationTime ||
+          creation_time >= (current_time - mutable_cf_options.ttl)) {
+        break;
       }
+      total_size -= f->compensated_file_size;
+      inputs[0].files.push_back(f);
     }
   }
 
@@ -100,7 +96,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
                      "[%s] FIFO compaction: picking file %" PRIu64
                      " with creation time %" PRIu64 " for deletion",
                      cf_name.c_str(), f->fd.GetNumber(),
-                     f->fd.table_reader->GetTableProperties()->creation_time);
+                     f->TryGetFileCreationTime());
   }
 
   Compaction* c = new Compaction(
diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc
index 854844396..7807d4737 100644
--- a/db/db_compaction_test.cc
+++ b/db/db_compaction_test.cc
@@ -5151,6 +5151,31 @@ TEST_P(DBCompactionTestWithParam,
   }
 }
 
+TEST_F(DBCompactionTest, FifoCompactionGetFileCreationTime) {
+  MockEnv mock_env(env_);
+  do {
+    Options options = CurrentOptions();
+    options.table_factory.reset(new BlockBasedTableFactory());
+    options.env = &mock_env;
+    options.ttl = static_cast<uint64_t>(24) * 3600;
+    options.compaction_style = kCompactionStyleFIFO;
+    constexpr size_t kNumFiles = 24;
+    options.max_open_files = 20;
+    constexpr size_t kNumKeysPerFile = 10;
+    DestroyAndReopen(options);
+    for (size_t i = 0; i < kNumFiles; ++i) {
+      for (size_t j = 0; j < kNumKeysPerFile; ++j) {
+        ASSERT_OK(Put(std::to_string(j), "value_" + std::to_string(i)));
+      }
+      ASSERT_OK(Flush());
+    }
+    mock_env.FakeSleepForMicroseconds(
+        static_cast<uint64_t>(1000 * 1000 * (1 + options.ttl)));
+    ASSERT_OK(Put("foo", "value"));
+    ASSERT_OK(Flush());
+  } while (ChangeOptions());
+}
+
 #endif // !defined(ROCKSDB_LITE)
 }  // namespace ROCKSDB_NAMESPACE
 
diff --git a/include/rocksdb/advanced_options.h b/include/rocksdb/advanced_options.h
index a72edbe05..fe6ce9250 100644
--- a/include/rocksdb/advanced_options.h
+++ b/include/rocksdb/advanced_options.h
@@ -645,7 +645,6 @@ struct AdvancedColumnFamilyOptions {
   bool report_bg_io_stats = false;
 
   // Files older than TTL will go through the compaction process.
-  // Pre-req: This needs max_open_files to be set to -1.
   // In Level: Non-bottom-level files older than TTL will go through the
   //           compation process.
   // In FIFO: Files older than TTL will be deleted.
@@ -673,7 +672,6 @@ struct AdvancedColumnFamilyOptions {
   // Supported in Level and FIFO compaction.
   // In FIFO compaction, this option has the same meaning as TTL and whichever
   // stricter will be used.
-  // Pre-req: max_open_file == -1.
   // unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60
   //
   // Values: